Compare perf benchmarks against a rolling median, not the last run #135

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]
	workflow_dispatch:

	permissions:
	contents: read

	jobs:
	test:
	name: Unit & integration tests
	runs-on: ubuntu-latest
	# Hard cap so a stalled Playwright download fails in minutes, not the 6h default.
	timeout-minutes: 20
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: recursive

	- uses: actions/setup-node@v5
	with:
	# Node 22 LTS: Node 24.16.0 hangs forever extracting the Playwright
	# browser zip (worked on 24.14.1; broke on a 24.x patch bump). See #123.
	node-version: 22
	cache: npm

	- name: Install dependencies
	run: npm ci

	- name: Install GPTK dependencies
	run: npm ci
	working-directory: Google-Photos-Toolkit

	# Unit tests (Vitest — pure JS, no browser)
	- name: Unit tests
	run: npm test

	# Build extension (required before integration tests)
	- name: Build extension
	run: npm run build

	# Integration tests require a headed browser; use Xvfb on Linux.
	# Cache the browser binaries so we only hit the CDN when the Playwright
	# version changes — keyed on the resolved @playwright/test version.
	- name: Resolve Playwright version
	id: pw-version
	run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"

	- name: Cache Playwright browsers
	id: pw-cache
	uses: actions/cache@v4
	with:
	path: ~/.cache/ms-playwright
	key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}

	# System deps (apt) must be installed on every run; they aren't cached.
	- name: Install Playwright system deps
	run: npx playwright install-deps chromium

	# Download the browser only on a cache miss. Wrap in timeout + retry:
	# the CDN download intermittently stalls at 100%, which previously hung
	# the job for the full 6h runner limit (see #123).
	- name: Install Playwright browser
	if: steps.pw-cache.outputs.cache-hit != 'true'
	run: \|
	for attempt in 1 2 3; do
	echo "::group::Playwright browser install (attempt $attempt)"
	if timeout 300 npx playwright install chromium; then
	echo "::endgroup::"
	exit 0
	fi
	echo "::endgroup::"
	echo "Attempt $attempt timed out or failed; retrying..."
	done
	echo "Playwright browser install failed after 3 attempts" >&2
	exit 1

	- name: Integration tests
	run: xvfb-run --auto-servernum npm run test:integration

	- name: Upload integration test results on failure
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: integration-test-results
	path: test-results/

	bench:
	name: Performance benchmarks
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v5

	- uses: actions/setup-node@v5
	with:
	node-version: 24
	cache: npm

	- name: Install dependencies
	run: npm ci

	- name: Run benchmarks
	run: npx vitest bench --run --outputJson bench-output.json

	# Restore the rolling history of recent main-branch runs.
	# restore-keys falls back to the latest matching entry when no exact hit.
	- name: Restore benchmark history
	uses: actions/cache/restore@v4
	with:
	path: tests/perf/bench-history.json
	key: bench-history-${{ github.sha }}
	restore-keys: bench-history-

	# Compares against the median of the last several main-branch runs
	# rather than just the last one — see tools/check-bench.mjs for why.
	- name: Check for regressions
	run: node tools/check-bench.mjs

	# Append this run to the rolling history (capped at the last 5) and
	# cache it under this SHA. Only runs on main so PRs always compare
	# against known-good history, not against another PR's numbers.
	- name: Update benchmark history
	if: github.ref == 'refs/heads/main'
	run: node tools/update-bench-history.mjs

	- name: Save benchmark history
	if: github.ref == 'refs/heads/main'
	uses: actions/cache/save@v4
	with:
	path: tests/perf/bench-history.json
	key: bench-history-${{ github.sha }}

Navigation Menu

Compare perf benchmarks against a rolling median, not the last run #135

Workflow file

Compare perf benchmarks against a rolling median, not the last run #135

Uh oh!

Workflow file for this run