fix(ci): run the full dashboard suite when a change reaches outside src/ (#37563)

The UI unit test job narrows a pull request to `vitest related <changed
files>`. `related` maps a file to the tests that import it, so a file no
test imports maps to nothing, and `--passWithNoTests` turns that empty
selection into a green job. package.json, package-lock.json, the Vitest,
Tailwind and TypeScript configs and tests/setupTests.ts are all in that
category even though each of them can change the behaviour of every test
in the suite, so a dashboard dependency bump merged having run no unit
tests at all and only got real coverage later, from the full run on the
push to litellm_internal_staging.

Keep `related` for the common case where a pull request only touches
files under src/, and fall back to the full suite as soon as one changed
file sits outside it. The decision lives in
.github/scripts/select_ui_test_scope.sh so it can be tested on its own,
next to the existing classify_changes.sh gate.

Author: yuneng-berri

.github/scripts/select_ui_test_scope.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+has_file=false
+has_file_outside_src=false
+while IFS= read -r file || [ -n "$file" ]; do
+  [ -n "$file" ] || continue
+  has_file=true
+  case "$file" in
+    src/*) ;;
+    *) has_file_outside_src=true ;;
+  esac
+done
+
+{ [ "$has_file" = true ] && [ "$has_file_outside_src" = false ]; } && echo related || echo full

.github/workflows/test-litellm-ui-unit.yml

@@ -65,22 +65,33 @@ jobs:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          if [ -n "$BASE_SHA" ]; then
-            merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
-            test -n "$merge_base"
-            git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
-            changed_files=()
-            while IFS= read -r f; do
-              changed_files+=("$f")
-            done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
-            if [ ${#changed_files[@]} -eq 0 ]; then
-              echo "No UI files changed in this PR; skipping unit tests."
-              exit 0
-            fi
-            echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
-            npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
-              --pool forks --poolOptions.forks.maxForks=14
-          else
+          full_suite() { npm run test -- --run --pool forks --poolOptions.forks.maxForks=14; }
+
+          if [ -z "$BASE_SHA" ]; then
             echo "Push to $GITHUB_REF_NAME: running the full suite"
-            npm run test -- --run --pool forks --poolOptions.forks.maxForks=14
+            full_suite
+            exit 0
+          fi
+
+          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$merge_base"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+          changed_files=()
+          while IFS= read -r f; do
+            changed_files+=("$f")
+          done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
+          if [ ${#changed_files[@]} -eq 0 ]; then
+            echo "No UI files changed in this PR; skipping unit tests."
+            exit 0
           fi
+
+          scope=$(printf '%s\n' "${changed_files[@]}" | bash "$GITHUB_WORKSPACE/.github/scripts/select_ui_test_scope.sh")
+          if [ "$scope" != related ]; then
+            echo "Pull request: ${#changed_files[@]} changed UI files reach outside src/, so related would miss their dependents; running the full suite"
+            full_suite
+            exit 0
+          fi
+
+          echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
+          npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
+            --pool forks --poolOptions.forks.maxForks=14

tests/test_litellm/test_select_ui_test_scope.py

@@ -0,0 +1,166 @@
+"""Regression tests for the UI unit-test scope decision.
+
+`.github/workflows/test-litellm-ui-unit.yml` narrows the dashboard's Vitest run to
+`vitest related <changed files>` so a pull request only pays for the tests it can
+affect. `related` resolves a file to the tests that import it, so a file no test
+imports resolves to nothing, and with `--passWithNoTests` the job then goes green
+without running a single test. `package.json`, `package-lock.json`, the Vitest and
+TypeScript configs and `tests/setupTests.ts` are all such files, so a dependency
+bump used to merge untested.
+
+`.github/scripts/select_ui_test_scope.sh` is the decision function that closes
+that hole: it prints `related` only when every changed file lives under `src/`,
+and `full` otherwise. These tests lock both the decision and the workflow step
+that consumes it, running the step's real shell against stubbed `gh`, `git` and
+`npm` so a regression shows up as the wrong Vitest command.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCOPE_SCRIPT = REPO_ROOT / ".github" / "scripts" / "select_ui_test_scope.sh"
+WORKFLOW = REPO_ROOT / ".github" / "workflows" / "test-litellm-ui-unit.yml"
+STEP_NAME = "Run UI unit tests (Vitest)"
+
+FULL_SUITE_ARGV = ["run", "test", "--", "--run", "--pool", "forks", "--poolOptions.forks.maxForks=14"]
+
+NON_SRC_FILES = [
+    "package.json",
+    "package-lock.json",
+    "vitest.config.ts",
+    "tsconfig.json",
+    "tests/setupTests.ts",
+    "next.config.mjs",
+]
+
+
+def scope(changed: list[str]) -> str:
+    result = subprocess.run(
+        ["bash", str(SCOPE_SCRIPT)],
+        input="\n".join(changed),
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout.strip()
+
+
+@pytest.mark.parametrize("changed_file", NON_SRC_FILES)
+def test_a_file_no_test_imports_selects_the_full_suite(changed_file: str) -> None:
+    assert scope([changed_file]) == "full"
+
+
+def test_src_only_changes_stay_on_related() -> None:
+    assert scope(["src/app/page.tsx", "src/lib/http/client.ts"]) == "related"
+
+
+def test_one_non_src_file_pulls_a_src_only_set_up_to_full() -> None:
+    assert scope(["src/app/page.tsx", "package-lock.json"]) == "full"
+
+
+def test_a_path_merely_prefixed_with_src_is_not_under_src() -> None:
+    assert scope(["srcipts/build.mjs"]) == "full"
+
+
+def test_an_empty_change_set_fails_open_to_the_full_suite() -> None:
+    assert scope([]) == "full"
+
+
+def _step_script() -> str:
+    workflow = yaml.safe_load(WORKFLOW.read_text())
+    steps = workflow["jobs"]["ui-unit-tests"]["steps"]
+    script = next(step["run"] for step in steps if step.get("name") == STEP_NAME)
+    resolved = script.replace("${{ github.repository }}", "BerriAI/litellm")
+    assert "${{" not in resolved, "the step uses an Actions expression this harness does not resolve"
+    return resolved
+
+
+def _stub(bin_dir: Path, name: str, body: str) -> None:
+    stub = bin_dir / name
+    stub.write_text(f"#!/usr/bin/env bash\n{body}\n")
+    stub.chmod(0o755)
+
+
+def _run_step(tmp_path: Path, changed: list[str], base_sha: str = "basesha") -> tuple[int, str, list[list[str]]]:
+    """Run the workflow step's real shell; return its status, stdout and every npm argv."""
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    changed_file = tmp_path / "changed.txt"
+    changed_file.write_text("".join(f"{name}\n" for name in changed))
+    npm_log = tmp_path / "npm.log"
+
+    _stub(bin_dir, "gh", 'echo "mergebasesha"')
+    _stub(bin_dir, "git", 'if [ "$1" = diff ]; then cat "$CHANGED_FILES"; fi')
+    _stub(bin_dir, "npm", 'printf "%s\\n" "$@" >>"$NPM_LOG"; printf "\\0" >>"$NPM_LOG"')
+
+    step = tmp_path / "step.sh"
+    step.write_text(_step_script())
+
+    env = dict(os.environ)
+    env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
+    env["BASE_SHA"] = base_sha
+    env["HEAD_SHA"] = "headsha"
+    env["GITHUB_WORKSPACE"] = str(REPO_ROOT)
+    env["GITHUB_REF_NAME"] = "litellm_internal_staging"
+    env["CHANGED_FILES"] = str(changed_file)
+    env["NPM_LOG"] = str(npm_log)
+
+    result = subprocess.run(
+        ["bash", "--noprofile", "--norc", "-eo", "pipefail", str(step)],
+        cwd=tmp_path,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    raw = npm_log.read_text() if npm_log.exists() else ""
+    invocations = [call.splitlines() for call in raw.split("\0") if call]
+    return result.returncode, result.stdout + result.stderr, invocations
+
+
+def _related_argv(changed: list[str]) -> list[str]:
+    return [
+        "run",
+        "test",
+        "--",
+        "related",
+        *changed,
+        "--run",
+        "--passWithNoTests",
+        "--pool",
+        "forks",
+        "--poolOptions.forks.maxForks=14",
+    ]
+
+
+def test_step_runs_related_for_a_src_only_pull_request(tmp_path: Path) -> None:
+    changed = ["src/app/page.tsx", "src/lib/http/client.ts"]
+    returncode, output, invocations = _run_step(tmp_path, changed)
+    assert returncode == 0, output
+    assert invocations == [_related_argv(changed)]
+
+
+@pytest.mark.parametrize("changed_file", NON_SRC_FILES)
+def test_step_runs_the_full_suite_when_a_changed_file_is_outside_src(tmp_path: Path, changed_file: str) -> None:
+    returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx", changed_file])
+    assert returncode == 0, output
+    assert invocations == [FULL_SUITE_ARGV]
+    assert "related" not in " ".join(invocations[0])
+
+
+def test_step_runs_the_full_suite_on_a_push(tmp_path: Path) -> None:
+    returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx"], base_sha="")
+    assert returncode == 0, output
+    assert invocations == [FULL_SUITE_ARGV]
+
+
+def test_step_runs_nothing_when_the_pull_request_touches_no_dashboard_file(tmp_path: Path) -> None:
+    returncode, output, invocations = _run_step(tmp_path, [])
+    assert returncode == 0, output
+    assert invocations == []