ci: enforce CLI shard SLA

Author: HavenDV

.github/workflows/main.yml

@@ -93,6 +93,12 @@ jobs:
           path: artifacts/test-shards-summary
           retention-days: 30
 
+      - name: Enforce shard SLA
+        run: python3 scripts/check-test-shard-sla.py
+          artifacts/test-shards-summary/shards.csv
+          --expected-shards 8
+          --max-elapsed-seconds 720
+
   test-cli:
     runs-on: ubuntu-latest
     strategy:

scripts/check-test-shard-sla.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import csv
+from pathlib import Path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Enforce completeness and elapsed-time SLAs for test shards.")
+    parser.add_argument("shards_csv", type=Path)
+    parser.add_argument("--expected-shards", type=int, required=True)
+    parser.add_argument("--max-elapsed-seconds", type=float, required=True)
+    args = parser.parse_args()
+
+    with args.shards_csv.open(encoding="utf-8", newline="") as stream:
+        rows = list(csv.DictReader(stream))
+
+    errors: list[str] = []
+    rows_by_shard: dict[int, dict[str, str]] = {}
+    for row in rows:
+        index = int(row["shard"])
+        if index in rows_by_shard:
+            errors.append(f"shard {index}: duplicate diagnostics")
+        rows_by_shard[index] = row
+
+    expected = set(range(args.expected_shards))
+    actual = set(rows_by_shard)
+    for index in sorted(expected - actual):
+        errors.append(f"shard {index}: diagnostics are missing")
+    for index in sorted(actual - expected):
+        errors.append(f"shard {index}: unexpected diagnostics")
+
+    for index in sorted(expected & actual):
+        row = rows_by_shard[index]
+        assigned = int(row["assigned"])
+        reported = int(row["reported"])
+        passed = int(row["passed"])
+        failed = int(row["failed"])
+
+        if reported != assigned:
+            errors.append(f"shard {index}: reported {reported} of {assigned} assigned tests")
+        if failed:
+            errors.append(f"shard {index}: {failed} tests failed")
+        if passed != reported:
+            errors.append(f"shard {index}: only {passed} of {reported} reported tests passed")
+
+        elapsed_value = row.get("elapsed_seconds", "")
+        if not elapsed_value:
+            errors.append(f"shard {index}: elapsed time is missing")
+            continue
+        elapsed_seconds = float(elapsed_value)
+        if elapsed_seconds > args.max_elapsed_seconds:
+            errors.append(
+                f"shard {index}: elapsed {elapsed_seconds:.1f}s exceeds "
+                f"{args.max_elapsed_seconds:.1f}s SLA"
+            )
+
+    if errors:
+        for error in errors:
+            print(f"::error title=CLI integration shard SLA::{error}")
+        raise SystemExit(1)
+
+    print(
+        f"All {args.expected_shards} shards reported complete passing results within "
+        f"{args.max_elapsed_seconds:.1f}s."
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/run-test-shard.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+shard_started_at=$SECONDS
+
 if [[ $# -lt 3 || $# -gt 4 ]]; then
   echo "Usage: $0 <project> <zero-based-shard-index> <shard-count> [configuration]" >&2
   exit 2
@@ -100,6 +102,7 @@ printf 'Running shard %d of %d with %d of %d tests.\n' \
   "${#filters[@]}" \
   "${#tests[@]}"
 
+set +e
 dotnet test "$project" \
   --configuration "$configuration" \
   --nologo \
@@ -108,3 +111,8 @@ dotnet test "$project" \
   --results-directory "$shard_artifacts" \
   --no-build \
   --filter "$filter"
+test_exit_code=$?
+set -e
+
+printf 'elapsed_seconds\t%s\n' "$((SECONDS - shard_started_at))" >> "$shard_artifacts/metadata.tsv"
+exit "$test_exit_code"

scripts/summarize-test-shards.py

@@ -28,6 +28,7 @@ class ShardResult:
     index: int
     membership: tuple[str, ...]
     tests: tuple[TestResult, ...]
+    elapsed_seconds: float | None
 
 
 def parse_duration(value: str) -> float:
@@ -70,6 +71,19 @@ def read_trx(path: Path) -> tuple[TestResult, ...]:
     return tuple(results)
 
 
+def read_elapsed_seconds(path: Path) -> float | None:
+    if not path.exists():
+        return None
+
+    with path.open(encoding="utf-8", newline="") as stream:
+        metadata = {
+            row["key"]: row["value"]
+            for row in csv.DictReader(stream, delimiter="\t")
+        }
+    value = metadata.get("elapsed_seconds")
+    return float(value) if value is not None else None
+
+
 def load_shards(input_directory: Path) -> tuple[ShardResult, ...]:
     shards = []
     for membership_path in sorted(input_directory.rglob("membership.txt")):
@@ -81,7 +95,15 @@ def load_shards(input_directory: Path) -> tuple[ShardResult, ...]:
         )
         trx_path = membership_path.with_name("results.trx")
         tests = read_trx(trx_path) if trx_path.exists() else ()
-        shards.append(ShardResult(index=index, membership=membership, tests=tests))
+        elapsed_seconds = read_elapsed_seconds(membership_path.with_name("metadata.tsv"))
+        shards.append(
+            ShardResult(
+                index=index,
+                membership=membership,
+                tests=tests,
+                elapsed_seconds=elapsed_seconds,
+            )
+        )
     return tuple(sorted(shards, key=lambda shard: shard.index))
 
 
@@ -104,7 +126,15 @@ def write_summary(output_directory: Path, shards: tuple[ShardResult, ...], expec
         passed = sum(test.outcome == "Passed" for test in shard.tests)
         failed = sum(test.outcome == "Failed" for test in shard.tests)
         shard_rows.append(
-            (shard.index, len(shard.membership), len(shard.tests), passed, failed, f"{duration:.3f}")
+            (
+                shard.index,
+                len(shard.membership),
+                len(shard.tests),
+                passed,
+                failed,
+                f"{duration:.3f}",
+                f"{shard.elapsed_seconds:.3f}" if shard.elapsed_seconds is not None else "",
+            )
         )
         for name in shard.membership:
             result = result_by_name.get(name)
@@ -121,7 +151,15 @@ def write_summary(output_directory: Path, shards: tuple[ShardResult, ...], expec
 
     write_csv(
         output_directory / "shards.csv",
-        ("shard", "assigned", "reported", "passed", "failed", "duration_seconds"),
+        (
+            "shard",
+            "assigned",
+            "reported",
+            "passed",
+            "failed",
+            "duration_seconds",
+            "elapsed_seconds",
+        ),
         shard_rows,
     )
     write_csv(
@@ -144,12 +182,14 @@ def write_summary(output_directory: Path, shards: tuple[ShardResult, ...], expec
         f"Discovered artifacts for **{len(shards)} of {expected_shards}** shards. ",
         f"Assigned tests: **{total_assigned}**. Reported results: **{total_reported}**.",
         "",
-        "| Shard | Assigned | Reported | Passed | Failed | Test duration |",
-        "| ---: | ---: | ---: | ---: | ---: | ---: |",
+        "| Shard | Assigned | Reported | Passed | Failed | Test duration | Elapsed |",
+        "| ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
     ]
-    for index, assigned, reported, passed, failed, duration in shard_rows:
+    for index, assigned, reported, passed, failed, duration, elapsed in shard_rows:
+        elapsed_display = f"{float(elapsed) / 60:.1f} min" if elapsed else "missing"
         lines.append(
-            f"| {index} | {assigned} | {reported} | {passed} | {failed} | {float(duration) / 60:.1f} min |"
+            f"| {index} | {assigned} | {reported} | {passed} | {failed} | "
+            f"{float(duration) / 60:.1f} min | {elapsed_display} |"
         )
     if missing_shards:
         lines.extend(("", f"Missing shard artifacts: {', '.join(map(str, missing_shards))}."))