[ci] start tracking retried tests (#15326)

## Change Description

Start capturing retried failed tests as a proxy for flaky tests. 

This will allow us to follow up with additional tracking of the most
common failures, make lists of links to similar test failures for bulk
analysis, etc.

Potentially, we could even add a "YOLO merge anyway" button as an
alternative to Retry, but let's start with a simple tracking change and
see how far we can get with just improving the tests.

Example row recorded by a test Retry:

```
mysql> select * from retried_tests;
+----+----------+--------+--------------+--------+-----------+-----------+-------------------+------------------------+------------------------------------------+------------+---------------------+
| id | batch_id | job_id | job_name     | state  | exit_code | pr_number | target_branch     | source_branch          | source_sha                               | retried_by | retried_at          |
+----+----------+--------+--------------+--------+-----------+-----------+-------------------+------------------------+------------------------------------------+------------+---------------------+
|  1 |  8370753 |    186 | test_batch_4 | Failed |         1 |     15320 | hail-is/hail:main | refresh-cert-on-deploy | 1518edb | chrisl     | 2026-03-10 19:03:09 |
+----+----------+--------+--------------+--------+-----------+-----------+-------------------+------------------------+------------------------------------------+------------+---------------------+
1 row in set (0.01 sec)
```

## Security Assessment

- This change potentially impacts the Hail Batch instance as deployed by
Broad Institute in GCP

### Impact Rating

- This change has a low security impact

### Impact Description

Store well defined data types (that are already being stored elsewhere
in the db) in a new table, as part of an existing development workflow.

### Appsec Review

- [x] Required: The impact has been assessed and approved by appsec

Author: cjllanwarne

build.yaml

@@ -2093,6 +2093,9 @@ steps:
       - name: add-service-rate-limits
         script: /io/sql/006-add-service-rate-limits.sql
         online: true
+      - name: retried-tests
+        script: /io/sql/007-retried-tests.sql
+        online: true
     inputs:
       - from: /repo/ci/sql
         to: /io/sql

ci/ci/ci.py

@@ -325,7 +325,7 @@ def storage_uri_to_url(uri: str) -> str:
     return uri
 
 
-async def retry_pr(wb: WatchedBranch, pr: PR, request: web.Request):
+async def retry_pr(wb: WatchedBranch, pr: PR, request: web.Request, userdata: UserData):
     app = request.app
     session = await aiohttp_session.get_session(request)
 
@@ -341,6 +341,27 @@ async def retry_pr(wb: WatchedBranch, pr: PR, request: web.Request):
 
     batch_id = pr.batch.id
     db = app[AppKeys.DB]
+
+    async for job in pr.batch.jobs():
+        if job['state'] in ('Failed', 'Error'):
+            await db.execute_insertone(
+                '''INSERT INTO retried_tests
+                   (batch_id, job_id, job_name, state, exit_code, pr_number, target_branch, source_branch, source_sha, retried_by)
+                   VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)''',
+                (
+                    job['batch_id'],
+                    job['job_id'],
+                    job.get('name'),
+                    job['state'],
+                    job.get('exit_code'),
+                    pr.number,
+                    wb.branch.short_str(),
+                    pr.source_branch.name,
+                    pr.source_sha,
+                    userdata['username'],
+                ),
+            )
+
     await db.execute_insertone('INSERT INTO invalidated_batches (batch_id) VALUES (%s);', batch_id)
     await wb.notify_batch_changed(
         db, app[AppKeys.BATCH_CLIENT], app[AppKeys.GH_CLIENT], app[AppKeys.FROZEN_MERGE_DEPLOY]
@@ -353,10 +374,10 @@ async def retry_pr(wb: WatchedBranch, pr: PR, request: web.Request):
 @routes.post('/watched_branches/{watched_branch_index}/pr/{pr_number}/retry')
 @web_security_headers
 @auth.authenticated_developers_only(redirect=False)
-async def post_retry_pr(request: web.Request, _) -> NoReturn:
+async def post_retry_pr(request: web.Request, userdata: UserData) -> NoReturn:
     wb, pr = wb_and_pr_from_request(request)
 
-    await asyncio.shield(retry_pr(wb, pr, request))
+    await asyncio.shield(retry_pr(wb, pr, request, userdata))
     raise web.HTTPFound(deploy_config.external_url('ci', f'/watched_branches/{wb.index}/pr/{pr.number}'))
 
 

ci/sql/007-retried-tests.sql

@@ -0,0 +1,18 @@
+CREATE TABLE retried_tests (
+  id BIGINT NOT NULL AUTO_INCREMENT,
+  batch_id BIGINT NOT NULL,
+  job_id INT NOT NULL,
+  job_name VARCHAR(255),
+  state VARCHAR(50) NOT NULL,
+  exit_code INT,
+  pr_number INT NOT NULL,
+  target_branch VARCHAR(255) NOT NULL,
+  source_branch VARCHAR(255) NOT NULL,
+  source_sha VARCHAR(40) NOT NULL,
+  retried_by VARCHAR(255) NOT NULL,
+  retried_at TIMESTAMP NOT NULL DEFAULT (UTC_TIMESTAMP),
+  PRIMARY KEY (id),
+  INDEX retried_tests_batch_id (batch_id),
+  INDEX retried_tests_pr_number (pr_number),
+  INDEX retried_tests_job_name (job_name)
+) ENGINE = InnoDB;

ci/sql/estimated-current.sql

@@ -39,3 +39,22 @@ CREATE TABLE IF NOT EXISTS `deployed_services` (
 INSERT INTO `active_namespaces` (`namespace`) VALUES (`default`);
 INSERT INTO `deployed_services` (`namespace`, `service`) VALUES
 ('default', 'auth'), ('default', 'batch'), ('default', 'batch-driver'), ('default', 'ci');
+
+CREATE TABLE retried_tests (
+  id BIGINT NOT NULL AUTO_INCREMENT,
+  batch_id BIGINT NOT NULL,
+  job_id INT NOT NULL,
+  job_name VARCHAR(255),
+  state VARCHAR(50) NOT NULL,
+  exit_code INT,
+  pr_number INT NOT NULL,
+  target_branch VARCHAR(255) NOT NULL,
+  source_branch VARCHAR(255) NOT NULL,
+  source_sha VARCHAR(40) NOT NULL,
+  retried_by VARCHAR(255) NOT NULL,
+  retried_at TIMESTAMP NOT NULL DEFAULT (UTC_TIMESTAMP),
+  PRIMARY KEY (id),
+  INDEX retried_tests_batch_id (batch_id),
+  INDEX retried_tests_pr_number (pr_number),
+  INDEX retried_tests_job_name (job_name)
+) ENGINE = InnoDB;