Skip to content
Cypress e2e Test

Cypress e2e Test #26558

Sign in to view logs

Cypress e2e Test

Cypress e2e Test #26558

Workflow file for this run

.github/workflows/cypress-e2e-test.yml at 5ddaecf
name: Cypress e2e Test
# =============================================================================
# E2E Test Workflow with Cluster Failover and Smart Test Selection
# =============================================================================
#
# TRIGGERS:
# - Automatically after "Test" workflow completes on PRs
# - Manually via workflow_dispatch (Actions tab → Run workflow)
#
# CLUSTER FAILOVER:
# Primary: dash-e2e-int (checked first via DSC health)
# Secondary: dash-e2e (used if primary is unhealthy)
# Health Check: Logs into cluster → checks DSC conditions (Available, Degraded, odh-dashboardReady)
#
# TEST SELECTION (priority order):
# Default (always run):
# - @ci-dashboard-regression-tags
#
# 1. Manual input (workflow_dispatch):
# Enter tags in 'additional_tags' field: @Pipelines,@Workbenches
# Check 'skip_default' to omit @ci-dashboard-regression-tags
# Check 'skip_auto_detect' to omit Turbo-based package detection
#
# 2. PR labels (test:* pattern):
# Add labels with 'test:' prefix to your PR:
# test:Pipelines → @Pipelines
# test:ModelServing → @ModelServing
# test:Workbenches → @Workbenches
# Any 'test:<TagName>' label maps to '@<TagName>' Cypress grep tag
#
# 3. Auto-detected from PR changes (always additive):
# Turbo detects changed packages → reads "e2eCiTags" from package.json
# Git diff detects changed frontend sub-areas → inline mapping resolves tags
# All auto-detected tags are consolidated into ONE additional matrix job
#
# To add auto-detection for a package:
# Add "e2eCiTags": ["@YourTagCI"] to the package's package.json
# To add auto-detection for a frontend area:
# Add an entry to .github/frontend-ci-tags.json
#
# LIMITS:
# - Max 5 additional tags for labels/manual (prevents runner exhaustion)
# - Auto-detected tags are consolidated into 1 job (no limit needed)
# - 10 runners shared across 30+ devs
#
# REQUIRED SECRETS:
# PRIMARY: OC_SERVER_PRIMARY, OCP_CONSOLE_URL_PRIMARY, ODH_DASHBOARD_URL_PRIMARY
# SECONDARY: OC_SERVER, OCP_CONSOLE_URL, ODH_DASHBOARD_URL
# AUTH: GITLAB_TOKEN, GITLAB_TEST_VARS_URL, ODH_NAMESPACES
#
# After downloading test-variables.yml, e2e-tests registers GitHub Actions
# masks for AWS keys and other credentials so Cypress [EXEC] logs cannot leak them.
# =============================================================================
on:
workflow_run:
workflows: ["Test"]
types: [completed]
workflow_dispatch:
inputs:
additional_tags:
description: 'Extra test tags (e.g., @Pipelines,@Workbenches)'
required: false
default: ''
type: string
skip_default:
description: 'Skip default @ci-dashboard-regression-tags'
required: false
default: false
type: boolean
skip_auto_detect:
description: 'Skip auto-detection of changed packages'
required: false
default: false
type: boolean
run_id:
description: 'Test workflow run ID to reuse artifacts from (skips rebuild if provided)'
required: false
default: ''
type: string
concurrency:
group: e2e-${{ github.event.workflow_run.head_branch || github.ref }}
cancel-in-progress: true
permissions:
contents: read
pull-requests: read
actions: read
statuses: write
env:
NODE_VERSION: 22
DO_NOT_TRACK: 1
# =============================================================================
# JOBS
# =============================================================================
jobs:
# ---------------------------------------------------------------------------
# Cluster Selection - Health check with automatic failover
# ---------------------------------------------------------------------------
select-cluster:
if: >-
github.event_name == 'workflow_dispatch' ||
(github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success')
runs-on: self-hosted
outputs:
cluster_name: ${{ steps.select.outputs.cluster_name }}
steps:
- name: Download test credentials
run: |
echo "🔧 Downloading test credentials for cluster health check..."
curl -fk -H "Authorization: Bearer ${{ secrets.GITLAB_TOKEN }}" \
"${{ secrets.GITLAB_TEST_VARS_URL }}" \
-o /tmp/test-variables.yml
echo "✅ Downloaded test credentials"
- name: Select healthy cluster
id: select
env:
PRIMARY_SERVER: ${{ secrets.OC_SERVER_PRIMARY }}
PRIMARY_DASHBOARD: ${{ secrets.ODH_DASHBOARD_URL_PRIMARY }}
SECONDARY_SERVER: ${{ secrets.OC_SERVER }}
SECONDARY_DASHBOARD: ${{ secrets.ODH_DASHBOARD_URL }}
run: |
# Prevent stale/corrupt kubeconfig from blocking oc login
echo "🧹 Removing stale kubeconfig to prevent corrupt config blocking login..."
rm -f ~/.kube/config 2>/dev/null || true
# Extract credentials from test-variables.yml
TEST_VARS_FILE="/tmp/test-variables.yml"
OC_USERNAME=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "USERNAME:" | head -1 | sed 's/.*USERNAME: //' | tr -d ' ')
OC_PASSWORD=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "PASSWORD:" | head -1 | sed 's/.*PASSWORD: //' | tr -d ' ')
echo "::add-mask::$OC_PASSWORD"
echo "::add-mask::$OC_USERNAME"
# Check DSC health by logging in and verifying conditions
check_dsc_health() {
local server_url="$1"
local cluster_name="$2"
[[ -z "$server_url" ]] && echo " ❌ Server URL is empty" && return 1
echo " 🔗 Attempting login to: $server_url"
# Try to login
LOGIN_OUTPUT=$(oc login -u "$OC_USERNAME" -p "$OC_PASSWORD" --server="$server_url" --insecure-skip-tls-verify 2>&1) || true
if ! oc whoami > /dev/null 2>&1; then
echo " ❌ Failed to login to $cluster_name"
echo " 📝 Login output: $LOGIN_OUTPUT" | head -5
return 1
fi
echo " ✅ Login successful"
# Get DSC status with full output for debugging
echo " 🔍 Fetching DataScienceCluster status..."
DSC_JSON=$(oc get datasciencecluster -o json 2>&1)
DSC_EXIT_CODE=$?
if [[ $DSC_EXIT_CODE -ne 0 ]]; then
echo " ❌ Failed to get DSC (exit code: $DSC_EXIT_CODE)"
echo " 📝 Output: $DSC_JSON" | head -5
return 1
fi
if [[ -z "$DSC_JSON" || "$DSC_JSON" == "null" || "$DSC_JSON" == '{"apiVersion":"datasciencecluster.opendatahub.io/v1","items":[],"kind":"List","metadata":{"resourceVersion":""}}' ]]; then
echo " ❌ No DataScienceCluster found on $cluster_name"
return 1
fi
# Print DSC name and status for debugging
DSC_NAME=$(echo "$DSC_JSON" | jq -r '.items[0].metadata.name // "unknown"')
echo " 📦 DSC Name: $DSC_NAME"
# Check phase - this is the most reliable indicator
PHASE=$(echo "$DSC_JSON" | jq -r '.items[0].status.phase // "Unknown"')
echo " 📊 DSC Phase: $PHASE"
# Print all conditions for debugging
echo " 📋 DSC Conditions:"
echo "$DSC_JSON" | jq -r '.items[0].status.conditions[]? | " - \(.type): \(.status) (\(.reason // "no reason"))"' 2>/dev/null || echo " (no conditions found)"
# If phase is Ready, cluster is healthy
if [[ "$PHASE" == "Ready" ]]; then
echo " ✅ DSC is Ready!"
return 0
fi
# Phase not Ready - check conditions for more detail
AVAILABLE=$(echo "$DSC_JSON" | jq -r '.items[0].status.conditions[] | select(.type=="Available") | .status' 2>/dev/null || echo "")
DEGRADED=$(echo "$DSC_JSON" | jq -r '.items[0].status.conditions[] | select(.type=="Degraded") | .status' 2>/dev/null || echo "")
# Fallback: if conditions show healthy even though phase isn't Ready
if [[ "$AVAILABLE" == "True" && "$DEGRADED" != "True" ]]; then
echo " ✅ Conditions look healthy despite phase=$PHASE"
return 0
fi
echo " ❌ DSC not healthy (Phase: $PHASE, Available: $AVAILABLE, Degraded: $DEGRADED)"
return 1
}
echo "🔍 Checking PRIMARY cluster (dash-e2e-int)..."
if check_dsc_health "$PRIMARY_SERVER" "dash-e2e-int"; then
echo "✅ PRIMARY cluster is healthy and ready"
echo "cluster_name=dash-e2e-int" >> $GITHUB_OUTPUT
else
echo ""
echo "⚠️ PRIMARY unavailable or not ready, trying SECONDARY (dash-e2e)..."
if check_dsc_health "$SECONDARY_SERVER" "dash-e2e"; then
echo "✅ SECONDARY cluster is healthy and ready"
echo "cluster_name=dash-e2e" >> $GITHUB_OUTPUT
else
echo ""
echo "❌ All clusters unavailable or unhealthy"
exit 1
fi
fi
# Clean up credentials file
rm -f /tmp/test-variables.yml
# ---------------------------------------------------------------------------
# Cleanup - Remove old test projects and resources (>2 hours old)
# ---------------------------------------------------------------------------
cleanup-old-resources:
needs: [select-cluster]
runs-on: self-hosted
continue-on-error: true # Don't fail the entire workflow if cleanup fails
env:
CLUSTER_NAME: ${{ needs.select-cluster.outputs.cluster_name }}
steps:
- name: Download test credentials
run: |
echo "🔧 Downloading test credentials for cleanup..."
curl -fk -H "Authorization: Bearer ${{ secrets.GITLAB_TOKEN }}" \
"${{ secrets.GITLAB_TEST_VARS_URL }}" \
-o /tmp/test-variables.yml
echo "✅ Downloaded test credentials"
- name: Login to selected cluster
env:
OC_SERVER_PRIMARY: ${{ secrets.OC_SERVER_PRIMARY }}
OC_SERVER_SECONDARY: ${{ secrets.OC_SERVER }}
run: |
TEST_VARS_FILE="/tmp/test-variables.yml"
# Extract credentials
OC_USERNAME=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "USERNAME:" | head -1 | sed 's/.*USERNAME: //' | tr -d ' ')
OC_PASSWORD=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "PASSWORD:" | head -1 | sed 's/.*PASSWORD: //' | tr -d ' ')
echo "::add-mask::$OC_PASSWORD"
echo "::add-mask::$OC_USERNAME"
# Determine cluster URL
if [ "$CLUSTER_NAME" = "dash-e2e-int" ]; then
CLUSTER_URL="$OC_SERVER_PRIMARY"
elif [ "$CLUSTER_NAME" = "dash-e2e" ]; then
CLUSTER_URL="$OC_SERVER_SECONDARY"
else
echo "❌ Unknown cluster: $CLUSTER_NAME"
exit 1
fi
# Remove stale kubeconfig
rm -f "$HOME/.kube/config" 2>/dev/null || true
# Login
echo "🔑 Logging in to $CLUSTER_NAME for cleanup..."
oc login -u "$OC_USERNAME" -p "$OC_PASSWORD" --server="$CLUSTER_URL" --insecure-skip-tls-verify > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "✅ Successfully logged in to $CLUSTER_NAME"
else
echo "❌ Failed to login - skipping cleanup"
exit 1
fi
- name: Cleanup old test projects and resources
run: |
echo "🧹 Cleaning up old test resources (>2 hours old) on $CLUSTER_NAME..."
# Calculate timestamp for 2 hours ago in seconds since epoch
CUTOFF_TIME=$(date -u -d '2 hours ago' +%s 2>/dev/null || date -u -v-2H +%s 2>/dev/null)
echo "📅 Cutoff time: $(date -u -d @${CUTOFF_TIME} 2>/dev/null || date -u -r ${CUTOFF_TIME} 2>/dev/null)"
DELETED_COUNT=0
SKIPPED_COUNT=0
STUCK_CLEANED=0
# Target namespaces with the dashboard label (added by Cypress createOpenShiftProject)
# This is more precise than pattern matching and catches all test projects
echo "🔍 Scanning for test projects with label 'opendatahub.io/dashboard=true'..."
while IFS='|' read -r project_name creation_time phase; do
# Convert creation timestamp to epoch seconds
PROJECT_TIME=$(date -u -d "$creation_time" +%s 2>/dev/null || date -u -j -f "%Y-%m-%dT%H:%M:%SZ" "$creation_time" +%s 2>/dev/null)
if [ -z "$PROJECT_TIME" ]; then
echo " ⚠️ Could not parse timestamp for $project_name - skipping"
SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
continue
fi
# Calculate age in hours
AGE_SECONDS=$(($(date -u +%s) - PROJECT_TIME))
AGE_HOURS=$((AGE_SECONDS / 3600))
# Handle stuck terminating namespaces (>2 hours in Terminating state)
if [ "$phase" = "Terminating" ] && [ "$PROJECT_TIME" -lt "$CUTOFF_TIME" ]; then
echo " 🔧 Force-cleaning stuck terminating namespace: $project_name (age: ${AGE_HOURS}h)..."
# Remove finalizers to unstick the namespace
oc patch namespace "$project_name" -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null && STUCK_CLEANED=$((STUCK_CLEANED + 1)) || true
continue
fi
# Delete active namespaces if older than 2 hours
if [ "$phase" = "Active" ] && [ "$PROJECT_TIME" -lt "$CUTOFF_TIME" ]; then
echo " 🗑️ Deleting $project_name (age: ${AGE_HOURS}h)..."
if oc delete project "$project_name" --wait=false 2>/dev/null; then
DELETED_COUNT=$((DELETED_COUNT + 1))
else
echo " ⚠️ Failed to delete $project_name"
fi
else
SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
fi
done < <(oc get projects -l opendatahub.io/dashboard=true -o json | jq -r '.items[] | "\(.metadata.name)|\(.metadata.creationTimestamp)|\(.status.phase // "Unknown")"')
echo ""
echo "📊 Project cleanup summary:"
echo " ✅ Deleted: $DELETED_COUNT projects"
echo " 🔧 Force-cleaned stuck: $STUCK_CLEANED terminating namespaces"
echo " ⏭️ Skipped: $SKIPPED_COUNT projects (too new)"
# Cleanup stuck terminating pods (>2 hours in Terminating state)
echo ""
echo "🧹 Cleaning up stuck terminating pods (>2 hours old)..."
PODS_CLEANED=0
while IFS='|' read -r ns pod; do
echo " 🔧 Force-deleting stuck pod: $pod in $ns..."
oc delete pod "$pod" -n "$ns" --grace-period=0 --force 2>/dev/null && PODS_CLEANED=$((PODS_CLEANED + 1)) || true
done < <(oc get pods -A -o json | jq -r --arg cutoff "$CUTOFF_TIME" '.items[] |
select(.status.phase == "Terminating" or (.metadata.deletionTimestamp != null)) |
select((.metadata.creationTimestamp | fromdateiso8601) < ($cutoff | tonumber)) |
"\(.metadata.namespace)|\(.metadata.name)"' 2>/dev/null)
echo " ✅ Force-deleted: $PODS_CLEANED stuck terminating pods"
# Cleanup orphaned PVCs in test namespaces
echo ""
echo "🧹 Cleaning up orphaned PVCs in test namespaces..."
PVC_COUNT=0
for ns in $(oc get ns -l opendatahub.io/dashboard=true -o name 2>/dev/null | sed 's|namespace/||'); do
OLD_PVCS=$(oc get pvc -n "$ns" -o json 2>/dev/null | jq -r --arg cutoff "$CUTOFF_TIME" '.items[] | select((.metadata.creationTimestamp | fromdateiso8601) < ($cutoff | tonumber)) | .metadata.name' 2>/dev/null || true)
if [ -n "$OLD_PVCS" ]; then
for pvc in $OLD_PVCS; do
echo " 🗑️ Deleting PVC $pvc in $ns..."
oc delete pvc "$pvc" -n "$ns" --wait=false 2>/dev/null && PVC_COUNT=$((PVC_COUNT + 1)) || true
done
fi
done
echo " ✅ Deleted: $PVC_COUNT orphaned PVCs"
# Clean up old DataScienceProjects in test namespaces
echo ""
echo "🧹 Cleaning up old DataScienceProjects in test namespaces..."
DSP_COUNT=0
for ns in $(oc get ns -l opendatahub.io/dashboard=true -o name 2>/dev/null | sed 's|namespace/||'); do
OLD_DSPS=$(oc get datascienceproject -n "$ns" -o json 2>/dev/null | jq -r --arg cutoff "$CUTOFF_TIME" '.items[] |
select((.metadata.creationTimestamp | fromdateiso8601) < ($cutoff | tonumber)) | .metadata.name' 2>/dev/null || true)
if [ -n "$OLD_DSPS" ]; then
for dsp in $OLD_DSPS; do
echo " 🗑️ Deleting DataScienceProject $dsp in $ns..."
oc delete datascienceproject "$dsp" -n "$ns" --wait=false 2>/dev/null && DSP_COUNT=$((DSP_COUNT + 1)) || true
done
fi
done
echo " ✅ Deleted: $DSP_COUNT DataScienceProjects"
echo ""
echo "✅ Cleanup complete on $CLUSTER_NAME"
- name: Cleanup credentials file
if: always()
run: rm -f /tmp/test-variables.yml
# ---------------------------------------------------------------------------
# Status - Set pending status on PR (independent - runs before cluster selection)
# ---------------------------------------------------------------------------
set-pending-status:
if: >-
github.event_name == 'workflow_dispatch' ||
(github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success')
runs-on: ubuntu-latest
steps:
- name: Set pending status
env:
GH_TOKEN: ${{ github.token }}
run: |
gh api repos/${{ github.repository }}/statuses/${{ github.event.workflow_run.head_sha || github.sha }} \
-f state=pending \
-f context="Cypress E2E Tests" \
-f description="E2E tests starting..." \
-f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
# ---------------------------------------------------------------------------
# Tag Resolution - Build test matrix from defaults + PR labels/input + auto-detection
# ---------------------------------------------------------------------------
get-test-tags:
needs: [select-cluster, cleanup-old-resources]
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.build.outputs.matrix }}
source: ${{ steps.build.outputs.source }}
packages: ${{ steps.discover-packages.outputs.packages }}
steps:
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Setup Node.js ${{ env.NODE_VERSION }}
uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # v4.3.0
with:
node-version: ${{ env.NODE_VERSION }}
cache: 'npm'
- name: Discover Cypress packages
id: discover-packages
uses: ./.github/actions/cypress-build-restore
with:
mode: discover
github-token: ${{ github.token }}
- name: Get PR labels
id: labels
if: github.event_name == 'workflow_run'
env:
GH_TOKEN: ${{ github.token }}
run: |
# Get PR number - try multiple methods for fork PR compatibility
PR_NUM="${{ github.event.workflow_run.pull_requests[0].number }}"
# Method 2: commits API (works for same-repo PRs)
if [[ -z "$PR_NUM" || "$PR_NUM" == "null" ]]; then
PR_NUM=$(gh api "repos/${{ github.repository }}/commits/${{ github.event.workflow_run.head_sha }}/pulls" \
--jq '.[0].number' 2>/dev/null || echo "")
fi
# Method 3: search API (works for fork PRs)
if [[ -z "$PR_NUM" || "$PR_NUM" == "null" ]]; then
PR_NUM=$(gh api "search/issues?q=repo:${{ github.repository }}+is:pr+is:open+sha:${{ github.event.workflow_run.head_sha }}" \
--jq '.items[0].number' 2>/dev/null || echo "")
fi
if [[ -n "$PR_NUM" && "$PR_NUM" != "null" ]]; then
LABELS=$(gh api "repos/${{ github.repository }}/issues/$PR_NUM/labels" \
--jq '[.[].name] | join(",")' 2>/dev/null || echo "")
echo "labels=$LABELS" >> $GITHUB_OUTPUT
echo "📋 PR #$PR_NUM labels: $LABELS"
else
echo "⚠️ Could not find PR number for SHA ${{ github.event.workflow_run.head_sha }}"
fi
- name: Detect changed areas
id: detect
if: inputs.skip_auto_detect != true
run: |
# =================================================================
# Smart Test Selection: Detect changed areas and resolve CI tags
#
# Layer 1: Turbo detects changed packages (including frontend workspace)
# → reads e2eCiTags from each package.json (self-service, teams opt in)
# Layer 2: For frontend changes detected by Layer 1, git diff identifies
# sub-areas → .github/frontend-ci-tags.json mapping resolves CI tags
# =================================================================
AUTO_TAGS=""
# --- Layer 1: Turbo-based package detection ---
echo "🔍 Running Turbo change detection..."
# Determine base ref for comparison
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
BASE_SHA="${{ github.event.workflow_run.pull_requests[0].base.sha || 'origin/main' }}"
else
BASE_SHA="origin/main"
fi
HEAD_SHA="${{ github.event.workflow_run.head_sha || github.sha }}"
echo " 📌 Comparing $BASE_SHA...$HEAD_SHA"
# Get changed packages from turbo (uses dependency graph)
CHANGED_PACKAGES=$(npx turbo run lint --dry=json --filter="...[$BASE_SHA...$HEAD_SHA]" 2>/dev/null \
| jq -r '.packages[]' 2>/dev/null || echo "")
if [[ -n "$CHANGED_PACKAGES" ]]; then
echo " 📦 Changed workspaces detected by Turbo:"
echo "$CHANGED_PACKAGES" | while read -r pkg; do echo " - $pkg"; done
# For each changed package, check for e2eCiTags in its package.json
for pkg_dir in packages/*/; do
pkg_name=$(jq -r '.name // empty' "$pkg_dir/package.json" 2>/dev/null)
if echo "$CHANGED_PACKAGES" | grep -qx "$pkg_name"; then
ci_tags=$(jq -r '.e2eCiTags[]? // empty' "$pkg_dir/package.json" 2>/dev/null)
if [[ -n "$ci_tags" ]]; then
for tag in $ci_tags; do
echo " ✅ $pkg_name → $tag"
AUTO_TAGS="$AUTO_TAGS $tag"
done
else
echo " ⏭️ $pkg_name (no e2eCiTags — defaults only)"
fi
fi
done
else
echo " ℹ️ No package changes detected by Turbo"
fi
# --- Layer 2: Frontend sub-area detection ---
# Turbo sees the entire frontend as one workspace. When it changes,
# use git diff to identify which sub-areas were modified.
# Turbo may report the frontend workspace as "//" or "odh-dashboard-frontend"
if echo "$CHANGED_PACKAGES" | grep -qxE "//|odh-dashboard-frontend"; then
echo ""
echo "🔍 Frontend changed — detecting sub-areas via git diff..."
# Load frontend directory → CI tag mapping from external JSON file
# To add a new area: edit .github/frontend-ci-tags.json
MAPPING_FILE=".github/frontend-ci-tags.json"
if [[ ! -f "$MAPPING_FILE" ]]; then
echo " ⚠️ $MAPPING_FILE not found — skipping frontend sub-area detection"
else
echo " 📄 Loaded mappings from $MAPPING_FILE"
# Get changed frontend files
CHANGED_FILES=$(git diff --name-only "$BASE_SHA"..."$HEAD_SHA" -- frontend/src/ 2>/dev/null || echo "")
if [[ -n "$CHANGED_FILES" ]]; then
# Scan pages/, concepts/, api/, routes/ using the same mapping
for src_dir in pages concepts api routes; do
DIRS=$(echo "$CHANGED_FILES" | grep "^frontend/src/$src_dir/" | \
sed "s|^frontend/src/$src_dir/||" | cut -d'/' -f1 | sort -u)
for dir in $DIRS; do
tag=$(jq -r --arg d "$dir" '.[$d] // empty' "$MAPPING_FILE")
if [[ -n "$tag" ]]; then
echo " ✅ $src_dir/$dir → $tag"
AUTO_TAGS="$AUTO_TAGS $tag"
fi
done
done
fi
fi
fi
# Deduplicate auto-detected tags
if [[ -n "$AUTO_TAGS" ]]; then
AUTO_TAGS=$(echo "$AUTO_TAGS" | tr ' ' '\n' | sort -u | tr '\n' ' ' | xargs)
echo ""
echo "🏷️ Auto-detected CI tags: $AUTO_TAGS"
else
echo ""
echo "ℹ️ No area-specific CI tags detected — defaults only"
fi
echo "auto_tags=$AUTO_TAGS" >> $GITHUB_OUTPUT
- name: Build test matrix
id: build
env:
INPUT_SKIP_DEFAULT: ${{ inputs.skip_default }}
INPUT_ADDITIONAL_TAGS: ${{ inputs.additional_tags }}
INPUT_SKIP_AUTO_DETECT: ${{ inputs.skip_auto_detect }}
STEP_AUTO_TAGS: ${{ steps.detect.outputs.auto_tags }}
STEP_LABELS: ${{ steps.labels.outputs.labels }}
run: |
# Configuration
MAX_EXTRA_TAGS=5 # Limit additional tags to prevent runner exhaustion (for labels/manual only)
# Defaults
TAGS=""
SOURCE="default"
if [[ "$INPUT_SKIP_DEFAULT" != "true" ]]; then
TAGS="@ci-dashboard-regression-tags"
else
echo "⏭️ Skipping default @ci-dashboard-regression-tags (skip_default=true)"
fi
EXTRA_COUNT=0
AUTO_DETECTED_ENTRY=""
# Priority 1: Manual input (workflow_dispatch)
if [[ -n "$INPUT_ADDITIONAL_TAGS" ]]; then
for tag in $(echo "$INPUT_ADDITIONAL_TAGS" | tr ',' ' '); do
if [[ ! "$tag" =~ ^@[a-zA-Z0-9_.:-]+$ ]]; then
echo "⚠️ Skipping invalid tag: $tag"
continue
fi
if [[ $EXTRA_COUNT -lt $MAX_EXTRA_TAGS ]]; then
TAGS="$TAGS,$tag"
EXTRA_COUNT=$((EXTRA_COUNT + 1))
fi
done
SOURCE="manual"
echo "📝 Added manual tags (limit: $MAX_EXTRA_TAGS)"
# Priority 2: PR labels (test:* pattern)
elif [[ -n "$STEP_LABELS" ]]; then
for label in $(echo "$STEP_LABELS" | tr ',' ' '); do
if [[ "$label" == test:* && $EXTRA_COUNT -lt $MAX_EXTRA_TAGS ]]; then
tag="@${label#test:}"
tag="${tag#@}" # Remove double @
tag="@$tag"
if [[ ! "$tag" =~ ^@[a-zA-Z0-9_.:-]+$ ]]; then
echo "⚠️ Skipping invalid tag from label: $tag"
continue
fi
TAGS="$TAGS,$tag"
EXTRA_COUNT=$((EXTRA_COUNT + 1))
SOURCE="pr-labels"
echo "🏷️ Label '$label' → $tag"
fi
done
fi
if [[ $EXTRA_COUNT -ge $MAX_EXTRA_TAGS ]]; then
echo "⚠️ Tag limit reached ($MAX_EXTRA_TAGS max). Some tags were not added."
fi
# Priority 3: Auto-detected from PR changes (additive, consolidated into ONE job)
AUTO_TAGS="$STEP_AUTO_TAGS"
if [[ "$INPUT_SKIP_AUTO_DETECT" == "true" ]]; then
echo "⏭️ Skipping auto-detection (skip_auto_detect=true)"
AUTO_TAGS=""
fi
if [[ -n "$AUTO_TAGS" ]]; then
# Remove any auto-detected tags that already appear in manual/label TAGS
# to prevent the same tests running in two separate matrix jobs
EXISTING_TAGS=$(echo "$TAGS" | tr ',' '\n' | sort -u)
FILTERED_AUTO=""
for auto_tag in $AUTO_TAGS; do
if [[ ! "$auto_tag" =~ ^@[a-zA-Z0-9_.:-]+$ ]]; then
echo "⚠️ Skipping invalid auto-detected tag: $auto_tag"
continue
fi
if echo "$EXISTING_TAGS" | grep -qx "$auto_tag"; then
echo "⏭️ Skipping $auto_tag from auto-detected (already in manual/label tags)"
else
FILTERED_AUTO="$FILTERED_AUTO $auto_tag"
fi
done
FILTERED_AUTO=$(echo "$FILTERED_AUTO" | xargs)
if [[ -n "$FILTERED_AUTO" ]]; then
# Consolidate remaining auto-detected tags into a single matrix entry
# Cypress grep treats space-separated tags as OR, so one job covers all areas
AUTO_DETECTED_ENTRY="$FILTERED_AUTO"
if [[ "$SOURCE" == "default" ]]; then
SOURCE="auto-detected"
else
SOURCE="$SOURCE+auto-detected"
fi
echo "🤖 Auto-detected tags (consolidated into 1 job): $AUTO_DETECTED_ENTRY"
else
echo "ℹ️ All auto-detected tags already covered by manual/label tags"
fi
fi
# Convert to JSON matrix (deduplicated)
if [[ -z "$TAGS" && -z "$AUTO_DETECTED_ENTRY" ]]; then
echo "❌ No tags to run. Provide additional_tags or disable skip options."
exit 1
fi
MATRIX=$(echo "$TAGS" | tr ',' '\n' | sort -u | grep -v '^$' | \
sed 's/^[^@]/@&/' | jq -Rc '[., inputs] | unique' | jq -sc 'add | unique')
# Append the consolidated auto-detected entry as a single matrix item
if [[ -n "$AUTO_DETECTED_ENTRY" ]]; then
MATRIX=$(echo "$MATRIX" | jq -c --arg entry "$AUTO_DETECTED_ENTRY" '. + [$entry] | unique')
fi
# Ensure compact JSON for GitHub Actions output
MATRIX=$(echo "$MATRIX" | jq -c '.')
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
echo "source=$SOURCE" >> $GITHUB_OUTPUT
echo "🧪 Final matrix: $MATRIX (source: $SOURCE)"
# ---------------------------------------------------------------------------
# Ensure Cypress Builds - Check which per-package artifacts exist
# ---------------------------------------------------------------------------
ensure-cypress-builds:
needs: [get-test-tags]
runs-on: ubuntu-latest
permissions:
contents: read
actions: read
outputs:
has-rebuilt: ${{ steps.ensure.outputs.has-rebuilt }}
missing-packages: ${{ steps.ensure.outputs.missing-packages }}
test-run-id: ${{ steps.ensure.outputs.test-run-id }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
persist-credentials: false
- name: Ensure Cypress builds
id: ensure
uses: ./.github/actions/cypress-build-restore
with:
mode: ensure
packages: ${{ needs.get-test-tags.outputs.packages }}
run-id: ${{ github.event.workflow_run.id || inputs.run_id || '' }}
github-token: ${{ github.token }}
repository: ${{ github.repository }}
# ---------------------------------------------------------------------------
# Build Missing Cypress Packages - Parallel matrix for any missing artifacts
# ---------------------------------------------------------------------------
build-missing-cypress:
needs: [ensure-cypress-builds]
if: needs.ensure-cypress-builds.outputs.has-rebuilt == 'true'
runs-on: ubuntu-latest
permissions:
contents: read
strategy:
fail-fast: false
matrix:
package: ${{ fromJson(needs.ensure-cypress-builds.outputs.missing-packages) }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
persist-credentials: false
- name: Setup Node.js ${{ env.NODE_VERSION }}
uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # v4.3.0
with:
node-version: ${{ env.NODE_VERSION }}
- name: Module caches
uses: ./.github/actions/module-caches
with:
node-version: ${{ env.NODE_VERSION }}
include-modules: 'true'
- name: Build and upload ${{ matrix.package.name }}
uses: ./.github/actions/cypress-build-restore
with:
mode: build
package-workspace: ${{ matrix.package.workspace }}
package-name: ${{ matrix.package.name }}
github-token: ${{ github.token }}
# ---------------------------------------------------------------------------
# E2E Tests - Run Cypress tests for each tag in parallel
# ---------------------------------------------------------------------------
e2e-tests:
needs: [select-cluster, set-pending-status, get-test-tags, ensure-cypress-builds, build-missing-cypress]
if: >-
!cancelled() && !failure() &&
needs.select-cluster.result == 'success'
runs-on: self-hosted
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
tag: ${{ fromJson(needs.get-test-tags.outputs.matrix) }}
env:
CLUSTER_NAME: ${{ needs.select-cluster.outputs.cluster_name }}
MATRIX_TAG: ${{ matrix.tag }}
steps:
- name: Validate tag format
run: |
if [[ -z "$MATRIX_TAG" ]]; then
echo "❌ MATRIX_TAG is empty or unset"
exit 1
fi
for tag in $MATRIX_TAG; do
if [[ ! "$tag" =~ ^@[a-zA-Z0-9_.:-]+$ ]]; then
echo "❌ Invalid tag format: $tag"
echo " Tags must match: ^@[a-zA-Z0-9_.:-]+$"
exit 1
fi
done
echo "✅ Tag format valid: $MATRIX_TAG"
- name: Check Disk Space
run: |
echo "📊 Checking available disk space..."
DISK_USAGE=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
DISK_AVAIL=$(df -h / | tail -1 | awk '{print $4}')
echo "💾 Disk usage: ${DISK_USAGE}% (${DISK_AVAIL} available)"
echo "DISK_USAGE=$DISK_USAGE" >> $GITHUB_ENV
if [ "$DISK_USAGE" -ge 95 ]; then
echo "❌ CRITICAL: Disk usage is ${DISK_USAGE}% - will attempt emergency cleanup"
echo "EMERGENCY_CLEANUP=true" >> $GITHUB_ENV
elif [ "$DISK_USAGE" -ge 90 ]; then
echo "⚠️ HIGH: Disk usage is ${DISK_USAGE}% - will attempt aggressive cleanup"
echo "EMERGENCY_CLEANUP=true" >> $GITHUB_ENV
elif [ "$DISK_USAGE" -ge 85 ]; then
echo "⚠️ WARNING: Disk usage is ${DISK_USAGE}% - cleanup recommended"
echo " The cleanup job will run after this workflow completes"
echo "EMERGENCY_CLEANUP=false" >> $GITHUB_ENV
else
echo "✅ Disk space OK (${DISK_USAGE}% used)"
echo "EMERGENCY_CLEANUP=false" >> $GITHUB_ENV
fi
- name: Emergency Cleanup (if disk space critical)
if: env.EMERGENCY_CLEANUP == 'true'
run: |
echo "🚨 EMERGENCY CLEANUP - Disk usage: ${DISK_USAGE}%"
RUNNER_USER=$(whoami)
HOME_DIR=$(eval echo "~$RUNNER_USER")
CURRENT_WORK_DIR="${{ github.workspace }}"
# Determine how aggressive to be based on disk usage
if [ "$DISK_USAGE" -ge 95 ]; then
AGE_THRESHOLD=1 # CRITICAL: Clean anything >1 day old
echo "⚠️ CRITICAL MODE: Cleaning files >1 day old"
else
AGE_THRESHOLD=7 # Normal: Clean anything >7 days old
echo "⚠️ AGGRESSIVE MODE: Cleaning files >7 days old"
fi
echo ""
if [ "$DISK_USAGE" -ge 95 ]; then
echo "🛡️ PARALLEL-SAFE PROTECTIONS (FAST MODE - disk critically full):"
echo " ✓ Current workspace (this job)"
echo " ✓ Active GitHub Actions Runner.Worker process directories"
echo " ⚡ Skipping slow checks (lsof, find) for speed"
else
echo "🛡️ PARALLEL-SAFE PROTECTIONS (THOROUGH MODE):"
echo " ✓ Current workspace (this job)"
echo " ✓ Active GitHub Actions Runner.Worker processes"
echo " ✓ Directories with open files (lsof with 5s timeout)"
echo " ✓ Directories accessed in last 10 minutes"
fi
echo ""
# Get list of ALL active work directories from currently running GitHub Actions jobs
# This is the safest way to avoid deleting directories from parallel PRs
echo "🔍 Detecting active work directories from parallel jobs (with timeout)..."
ACTIVE_WORK_DIRS=()
ACTIVE_JOBS=0
# Use faster method: check for active processes, then only protect their workspace
ACTIVE_PIDS=$(pgrep -f "Runner.Worker" -u "$RUNNER_USER" 2>/dev/null || true)
if [ -n "$ACTIVE_PIDS" ]; then
echo " Found active Runner.Worker processes: $ACTIVE_PIDS"
# Get working directories of active processes using lsof (much faster than find)
for pid in $ACTIVE_PIDS; do
ACTIVE_JOBS=$((ACTIVE_JOBS + 1))
# Get the CWD of this process
if [ -L "/proc/$pid/cwd" ]; then
WORK_CWD=$(readlink "/proc/$pid/cwd" 2>/dev/null || true)
if [[ "$WORK_CWD" == *"odh-dashboard"* ]]; then
# Extract the odh-dashboard directory path
WORK_DIR=$(echo "$WORK_CWD" | sed 's|/odh-dashboard/.*|/odh-dashboard|')
ACTIVE_WORK_DIRS+=("$WORK_DIR")
echo " 🛡️ Protected: $WORK_DIR (PID $pid)"
fi
fi
done
fi
echo " Found $ACTIVE_JOBS active runner(s) with ${#ACTIVE_WORK_DIRS[@]} protected work directory(ies)"
# Helper function to check if directory is in use by active runner
is_directory_in_use() {
local dir="$1"
# 1. Skip current workspace (absolute must)
if [[ "$dir" == "$CURRENT_WORK_DIR"* ]]; then
return 0 # In use (current job)
fi
# 2. Check if directory is in the active work dirs list (FAST)
for active_dir in "${ACTIVE_WORK_DIRS[@]}"; do
if [[ "$dir" == "$active_dir"* ]]; then
return 0 # In use (active job)
fi
done
# 3. In CRITICAL mode (disk ≥95%), skip slow checks - rely on active work dirs only
if [ "$DISK_USAGE" -ge 95 ]; then
return 1 # Not in active list, safe to delete (fast path)
fi
# 4. Normal mode: Do thorough checks
# Check for ANY processes using this directory (can be slow)
if timeout 5 lsof +D "$dir" 2>/dev/null | grep -q .; then
return 0 # In use (has open files)
fi
# 5. Check if directory was accessed very recently (last 10 minutes only)
if find "$dir" -maxdepth 0 -amin -10 2>/dev/null | grep -q .; then
return 0 # In use (very recent activity)
fi
return 1 # Not in use (safe to delete)
}
echo ""
echo "🗑️ Step 1: Cleaning Go upstream builds (age: >$AGE_THRESHOLD days)..."
UPSTREAM_CLEANED=0
find "$HOME_DIR"/actions-runner*/_work -type d -path "*/packages/*/upstream" -mtime +$AGE_THRESHOLD 2>/dev/null | while read upstream_dir; do
# Extract work_dir by going up to odh-dashboard parent
work_dir=$(echo "$upstream_dir" | sed 's|/odh-dashboard/.*|/odh-dashboard|')
if [ -n "$work_dir" ] && ! is_directory_in_use "$work_dir"; then
rm -rf "$upstream_dir" 2>/dev/null && echo " ✅ Cleaned: $upstream_dir" && UPSTREAM_CLEANED=$((UPSTREAM_CLEANED + 1)) || true
fi
done
echo ""
echo "🗑️ Step 2: Cleaning old work directories (age: >$AGE_THRESHOLD days, with multi-layer safety)..."
CLEANED_COUNT=0
SKIPPED_COUNT=0
find "$HOME_DIR"/actions-runner*/_work -maxdepth 1 -name "odh-dashboard" -type d -mtime +$AGE_THRESHOLD 2>/dev/null | while read work_dir; do
if [ -d "$work_dir" ]; then
if is_directory_in_use "$work_dir"; then
echo " ⏭️ Protected (in use): $work_dir"
SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
else
SIZE_BEFORE=$(du -sh "$work_dir" 2>/dev/null | awk '{print $1}')
if rm -rf "$work_dir" 2>/dev/null; then
echo " ✅ Cleaned $SIZE_BEFORE: $work_dir"
CLEANED_COUNT=$((CLEANED_COUNT + 1))
fi
fi
fi
done
echo " 📊 Cleaned: $CLEANED_COUNT, Protected: $SKIPPED_COUNT"
echo ""
echo "🗑️ Step 3: Cleaning Cypress artifacts (age: >$AGE_THRESHOLD days)..."
SCREENSHOTS_CLEANED=$(find "$HOME_DIR"/actions-runner*/_work -path "*/cypress/results/screenshots/*" -mtime +$AGE_THRESHOLD -delete -print 2>/dev/null | wc -l)
VIDEOS_CLEANED=$(find "$HOME_DIR"/actions-runner*/_work -path "*/cypress/results/videos/*" -mtime +$AGE_THRESHOLD -delete -print 2>/dev/null | wc -l)
echo " ✅ Cleaned $SCREENSHOTS_CLEANED screenshots, $VIDEOS_CLEANED videos"
echo ""
echo "🗑️ Step 4: Cleaning runner logs (age: >$AGE_THRESHOLD days)..."
LOGS_CLEANED=$(find "$HOME_DIR"/actions-runner*/_diag -name "*.log" -mtime +$AGE_THRESHOLD -delete -print 2>/dev/null | wc -l)
echo " ✅ Cleaned $LOGS_CLEANED log files"
echo ""
echo "🗑️ Step 5: Cleaning node_modules in old work dirs (age: >$AGE_THRESHOLD days)..."
find "$HOME_DIR"/actions-runner*/_work -type d -name "node_modules" -mtime +$AGE_THRESHOLD 2>/dev/null | while read nm_dir; do
# Extract work_dir by going up to odh-dashboard parent
work_dir=$(echo "$nm_dir" | sed 's|/odh-dashboard/.*|/odh-dashboard|')
if [ -n "$work_dir" ] && ! is_directory_in_use "$work_dir"; then
SIZE_BEFORE=$(du -sh "$nm_dir" 2>/dev/null | awk '{print $1}')
rm -rf "$nm_dir" 2>/dev/null && echo " ✅ Cleaned $SIZE_BEFORE node_modules: $nm_dir" || true
fi
done
echo ""
echo "🗑️ Step 6: Cleaning .turbo cache in old work dirs (age: >$AGE_THRESHOLD days)..."
find "$HOME_DIR"/actions-runner*/_work -type d -name ".turbo" -mtime +$AGE_THRESHOLD 2>/dev/null | while read turbo_dir; do
# Extract work_dir by going up to odh-dashboard parent
work_dir=$(echo "$turbo_dir" | sed 's|/odh-dashboard/.*|/odh-dashboard|')
if [ -n "$work_dir" ] && ! is_directory_in_use "$work_dir"; then
SIZE_BEFORE=$(du -sh "$turbo_dir" 2>/dev/null | awk '{print $1}')
rm -rf "$turbo_dir" 2>/dev/null && echo " ✅ Cleaned $SIZE_BEFORE .turbo: $turbo_dir" || true
fi
done
echo ""
echo "📊 Disk usage after emergency cleanup:"
DISK_USAGE_AFTER=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
DISK_AVAIL_AFTER=$(df -h / | tail -1 | awk '{print $4}')
echo "💾 Disk usage: ${DISK_USAGE_AFTER}% (${DISK_AVAIL_AFTER} available)"
FREED=$((DISK_USAGE - DISK_USAGE_AFTER))
if [ "$FREED" -gt 0 ]; then
echo "✅ Freed: ${FREED}% disk space"
else
echo "⚠️ Freed: 0% disk space (no files met age threshold)"
fi
# Show what's taking up space
echo ""
echo "📊 Top disk usage on runner:"
du -sh "$HOME_DIR"/actions-runner*/_work/* 2>/dev/null | sort -rh | head -5 || true
# Decide whether to fail or continue
if [ "$DISK_USAGE_AFTER" -ge 95 ]; then
if [ "$FREED" -gt 0 ]; then
echo "⚠️ WARNING: Still at ${DISK_USAGE_AFTER}% after cleanup, but freed ${FREED}%"
echo " Attempting to proceed - job may fail if more space is needed"
else
echo "❌ CRITICAL: Still at ${DISK_USAGE_AFTER}% after cleanup and freed 0%"
echo " All work directories are either:"
echo " - Currently in use by active jobs"
echo " - Created within the last $AGE_THRESHOLD day(s)"
echo ""
echo "🔍 Diagnosis - Active work directories:"
find "$HOME_DIR"/actions-runner*/_work -maxdepth 1 -name "odh-dashboard" -type d 2>/dev/null | while read work_dir; do
MTIME=$(stat -f %m "$work_dir" 2>/dev/null || stat -c %Y "$work_dir" 2>/dev/null || echo "0")
AGE_DAYS=$(( ($(date +%s) - MTIME) / 86400 ))
SIZE=$(du -sh "$work_dir" 2>/dev/null | awk '{print $1}')
echo " - $work_dir: $SIZE, age: ${AGE_DAYS} days"
done
echo ""
echo " Manual intervention required on runner $(hostname)"
exit 1
fi
elif [ "$DISK_USAGE_AFTER" -ge 90 ]; then
echo "⚠️ WARNING: Still at ${DISK_USAGE_AFTER}% after cleanup"
echo " Job will proceed but may fail due to space"
else
echo "✅ Cleanup successful - proceeding with tests"
fi
- name: Cleanup old test artifacts
continue-on-error: true
run: |
echo "🧹 Cleaning up old test artifacts (>2 days)..."
# Clean old Cypress results/screenshots/videos from workspace (>2 days old)
find ${{ github.workspace }}/packages/cypress/results -type f -mtime +2 -delete 2>/dev/null || true
find ${{ github.workspace }}/packages/cypress/screenshots -type f -mtime +2 -delete 2>/dev/null || true
find ${{ github.workspace }}/packages/cypress/videos -type f -mtime +2 -delete 2>/dev/null || true
# Note: .cache/Cypress is managed by actions/cache (workspace-local via CYPRESS_CACHE_FOLDER)
# and should not be cleaned here to avoid removing the Cypress binary mid-run
# Clean old temporary yaml files (>2 days old)
find /tmp -name "cypress-yaml-*.yaml" -type f -mtime +2 -delete 2>/dev/null || true
# Clean empty directories
find ${{ github.workspace }}/packages/cypress/results -type d -empty -delete 2>/dev/null || true
find ${{ github.workspace }}/packages/cypress/screenshots -type d -empty -delete 2>/dev/null || true
find ${{ github.workspace }}/packages/cypress/videos -type d -empty -delete 2>/dev/null || true
# Fix read-only envtest binaries and directories from previous BFF runs
# (setup-envtest downloads k8s binaries with 0555 permissions, which blocks actions/checkout cleanup)
chmod -R u+w ${{ github.workspace }}/packages/*/bff/bin ${{ github.workspace }}/packages/*/upstream/bff/bin 2>/dev/null || true
echo "✅ Cleanup complete (non-critical, continued on any errors)"
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
persist-credentials: false
# Fetch the mask script from the default branch into RUNNER_TEMP so a
# malicious PR cannot replace it. Do not use actions/checkout with a
# path under RUNNER_TEMP — checkout requires the destination to be
# under GITHUB_WORKSPACE, which fails on these self-hosted runners.
- name: Fetch trusted credential mask script
env:
GH_TOKEN: ${{ github.token }}
run: |
DEST_DIR="${{ runner.temp }}/trusted-credential-tools"
mkdir -p "$DEST_DIR"
DEST="$DEST_DIR/mask-cypress-test-secrets.sh"
API_URL="https://api.github.com/repos/${{ github.repository }}/contents/.github/scripts/mask-cypress-test-secrets.sh?ref=${{ github.event.repository.default_branch }}"
HTTP_CODE=$(curl -sS -L -o "$DEST" -w "%{http_code}" \
-H "Authorization: Bearer ${GH_TOKEN}" \
-H "Accept: application/vnd.github.raw" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"$API_URL" || true)
if [[ "$HTTP_CODE" != "200" || ! -s "$DEST" ]]; then
echo "⚠️ Trusted mask script not available from default branch (HTTP ${HTTP_CODE:-failed}); skipping Actions masks"
rm -f "$DEST"
else
echo "✅ Fetched mask script from ${{ github.event.repository.default_branch }}"
fi
- name: Module caches
uses: ./.github/actions/module-caches
with:
node-version: ${{ env.NODE_VERSION }}
- name: Setup Node.js ${{ env.NODE_VERSION }}
uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # v4.3.0
with:
node-version: ${{ env.NODE_VERSION }}
- name: Restore OpenShift CLI tarball cache
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
id: oc-cache
with:
path: ${{ runner.temp }}/oc.tar.gz
key: ${{ runner.os }}-oc-tarball-${{ env.OC_VERSION || '4.15.0' }}
- name: Download test configuration
run: |
echo "🔧 Downloading test configuration from GitLab..."
curl -fk -H "Authorization: Bearer ${{ secrets.GITLAB_TOKEN }}" \
"${{ secrets.GITLAB_TEST_VARS_URL }}" \
-o ${{ github.workspace }}/packages/cypress/test-variables.yml
echo "✅ Downloaded test configuration"
MASK_SCRIPT="${{ runner.temp }}/trusted-credential-tools/mask-cypress-test-secrets.sh"
if [[ -f "$MASK_SCRIPT" ]]; then
bash "$MASK_SCRIPT" "${{ github.workspace }}/packages/cypress/test-variables.yml"
else
echo "⚠️ Trusted mask script not available; skipping Actions masks"
fi
- name: Login to OpenShift cluster
env:
OC_SERVER_PRIMARY: ${{ secrets.OC_SERVER_PRIMARY }}
OC_SERVER_SECONDARY: ${{ secrets.OC_SERVER }}
run: |
TEST_VARS_FILE="${{ github.workspace }}/packages/cypress/test-variables.yml"
# Extract credentials based on test type
if [[ "$MATRIX_TAG" == "@NonAdmin" ]]; then
echo "🔑 Using non-admin credentials (TEST_USER_3) for @NonAdmin tests"
OC_USERNAME=$(grep -A 10 "^TEST_USER_3:" "$TEST_VARS_FILE" | grep "USERNAME:" | head -1 | sed 's/.*USERNAME: //' | tr -d ' ')
OC_PASSWORD=$(grep -A 10 "^TEST_USER_3:" "$TEST_VARS_FILE" | grep "PASSWORD:" | head -1 | sed 's/.*PASSWORD: //' | tr -d ' ')
else
OC_USERNAME=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "USERNAME:" | head -1 | sed 's/.*USERNAME: //' | tr -d ' ')
OC_PASSWORD=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "PASSWORD:" | head -1 | sed 's/.*PASSWORD: //' | tr -d ' ')
fi
echo "::add-mask::$OC_PASSWORD"
echo "::add-mask::$OC_USERNAME"
# Look up server URL based on selected cluster (avoids GitHub secret masking in outputs)
if [ "$CLUSTER_NAME" = "dash-e2e-int" ]; then
CLUSTER_URL="$OC_SERVER_PRIMARY"
elif [ "$CLUSTER_NAME" = "dash-e2e" ]; then
CLUSTER_URL="$OC_SERVER_SECONDARY"
else
echo "❌ Unknown or empty CLUSTER_NAME: '$CLUSTER_NAME'" >&2
echo "Expected 'dash-e2e-int' or 'dash-e2e'" >&2
exit 1
fi
if [ -z "$CLUSTER_URL" ]; then
echo "❌ CLUSTER_URL is empty for cluster '$CLUSTER_NAME'" >&2
echo "Check that OC_SERVER_PRIMARY/OC_SERVER secrets are configured" >&2
exit 1
fi
# Prevent stale/corrupt kubeconfig from blocking oc login
echo "🧹 Removing stale kubeconfig to prevent corrupt config blocking login..."
rm -f "$HOME/.kube/config" 2>/dev/null || true
echo "Logging in to OpenShift cluster ($CLUSTER_NAME)..."
oc login -u "$OC_USERNAME" -p "$OC_PASSWORD" --server="$CLUSTER_URL" --insecure-skip-tls-verify > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo "✅ Successfully logged in to $CLUSTER_NAME"
else
echo "❌ Failed to login to OpenShift cluster"
exit 1
fi
echo "KUBECONFIG=$HOME/.kube/config" >> $GITHUB_ENV
- name: Override namespace values
env:
DASHBOARD_URL_PRIMARY: ${{ secrets.ODH_DASHBOARD_URL_PRIMARY }}
DASHBOARD_URL_SECONDARY: ${{ secrets.ODH_DASHBOARD_URL }}
ODH_NAMESPACES: ${{ secrets.ODH_NAMESPACES }}
run: |
TEST_VARS_FILE="${{ github.workspace }}/packages/cypress/test-variables.yml"
# Look up dashboard URL based on selected cluster (secrets passed as step-level env for security)
if [ "$CLUSTER_NAME" = "dash-e2e-int" ]; then
DASHBOARD_URL="$DASHBOARD_URL_PRIMARY"
elif [ "$CLUSTER_NAME" = "dash-e2e" ]; then
DASHBOARD_URL="$DASHBOARD_URL_SECONDARY"
else
echo "❌ Unknown or empty CLUSTER_NAME: '$CLUSTER_NAME'" >&2
echo "Expected 'dash-e2e-int' or 'dash-e2e'" >&2
exit 1
fi
if [ -z "$DASHBOARD_URL" ]; then
echo "❌ DASHBOARD_URL is empty for cluster '$CLUSTER_NAME'" >&2
echo "Check that ODH_DASHBOARD_URL_PRIMARY/ODH_DASHBOARD_URL secrets are configured" >&2
exit 1
fi
# Mask dashboard URL to prevent exposure in logs
echo "::add-mask::$DASHBOARD_URL"
# Set dashboard URL for selected cluster
sed -i "s|^ODH_DASHBOARD_URL:.*|ODH_DASHBOARD_URL: $DASHBOARD_URL|" "$TEST_VARS_FILE"
# Export for e2e proxy
echo "ODH_DASHBOARD_URL=$DASHBOARD_URL" >> $GITHUB_ENV
if [ -n "$ODH_NAMESPACES" ]; then
echo "::add-mask::$ODH_NAMESPACES"
echo "📝 Overriding namespaces with ODH values..."
IFS=',' read -r OPERATOR_NS APPLICATIONS_NS NOTEBOOKS_NS OPERATOR_NAME PROJECT_NAME <<< "$ODH_NAMESPACES"
sed -i "s|^PRODUCT:.*|PRODUCT: ODH|" "$TEST_VARS_FILE"
sed -i "s|^OPERATOR_NAMESPACE:.*|OPERATOR_NAMESPACE: $OPERATOR_NS|" "$TEST_VARS_FILE"
sed -i "s|^APPLICATIONS_NAMESPACE:.*|APPLICATIONS_NAMESPACE: $APPLICATIONS_NS|" "$TEST_VARS_FILE"
sed -i "s|^MONITORING_NAMESPACE:.*|MONITORING_NAMESPACE: $APPLICATIONS_NS|" "$TEST_VARS_FILE"
sed -i "s|^NOTEBOOKS_NAMESPACE:.*|NOTEBOOKS_NAMESPACE: $NOTEBOOKS_NS|" "$TEST_VARS_FILE"
sed -i "s|^OPERATOR_NAME:.*|OPERATOR_NAME: $OPERATOR_NAME|" "$TEST_VARS_FILE"
sed -i "s|^ODH_DASHBOARD_PROJECT_NAME:.*|ODH_DASHBOARD_PROJECT_NAME: $PROJECT_NAME|" "$TEST_VARS_FILE"
echo "OC_PROJECT=$APPLICATIONS_NS" >> $GITHUB_ENV
echo "✅ Namespace configuration updated (OC_PROJECT=$APPLICATIONS_NS)"
else
echo "⚠️ ODH_NAMESPACES secret not set, skipping namespace override"
# Fall back to the default from the test config
OC_PROJECT=$(grep "^APPLICATIONS_NAMESPACE:" "$TEST_VARS_FILE" | sed 's/^APPLICATIONS_NAMESPACE: *//' | tr -d ' ')
if [ -n "$OC_PROJECT" ]; then
echo "OC_PROJECT=$OC_PROJECT" >> $GITHUB_ENV
echo "✅ OC_PROJECT set from test config defaults: $OC_PROJECT"
else
echo "⚠️ Could not determine APPLICATIONS_NAMESPACE from test config"
fi
fi
- name: Set test configuration
run: |
echo "CY_TEST_CONFIG=${{ github.workspace }}/packages/cypress/test-variables.yml" >> $GITHUB_ENV
- name: Restore Cypress builds
uses: ./.github/actions/cypress-build-restore
with:
mode: restore
run-id: ${{ needs.ensure-cypress-builds.outputs.test-run-id }}
packages: ${{ needs.get-test-tags.outputs.packages }}
github-token: ${{ github.token }}
- name: Setup Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
with:
go-version: '1.26.x'
- name: Prepare E2E dependencies
run: npm run prepare:e2e
- name: Run E2E Tests
env:
OC_SERVER_PRIMARY: ${{ secrets.OC_SERVER_PRIMARY }}
OC_SERVER_SECONDARY: ${{ secrets.OC_SERVER }}
CYPRESS_E2E_PROXY: 'true'
TAG_SOURCE: ${{ needs.get-test-tags.outputs.source }}
run: |
TAG="$MATRIX_TAG"
if [ ${#TAG} -gt 200 ]; then
TAG_DIR=$(echo -n "$TAG" | sha256sum | cut -c1-12)
echo "Tag too long for directory name (${#TAG} chars), using hash: ${TAG_DIR}"
else
TAG_DIR="$TAG"
fi
echo "TAG_DIR=${TAG_DIR}" >> $GITHUB_ENV
echo "Running E2E tests for $MATRIX_TAG..."
echo "Running tests against E2E proxy on port 4040"
echo "Tag source: $TAG_SOURCE"
export CY_RESULTS_DIR="${{ github.workspace }}/packages/cypress/results/${TAG_DIR}"
mkdir -p "$CY_RESULTS_DIR"
# Determine OC_SERVER based on cluster (for oc user switching in tests)
if [ "$CLUSTER_NAME" = "dash-e2e-int" ]; then
OC_SERVER="$OC_SERVER_PRIMARY"
elif [ "$CLUSTER_NAME" = "dash-e2e" ]; then
OC_SERVER="$OC_SERVER_SECONDARY"
else
echo "Unknown cluster: $CLUSTER_NAME, defaulting to OC_SERVER_PRIMARY"
OC_SERVER="$OC_SERVER_PRIMARY"
fi
# Cypress reads CYPRESS_* env vars natively — avoids arg escaping through concurrently
export CYPRESS_OC_SERVER="$OC_SERVER"
export CYPRESS_skipTags="@Bug @Maintain @NonConcurrent"
export CYPRESS_grepTags="$MATRIX_TAG"
export CYPRESS_grepFilterSpecs=true
export CYPRESS_video=true
export CYPRESS_screenshotsFolder="$CY_RESULTS_DIR/screenshots"
export CYPRESS_videosFolder="$CY_RESULTS_DIR/videos"
if [[ "$MATRIX_TAG" == "@NonAdmin" ]]; then
export CYPRESS_IS_NON_ADMIN_RUN=true
echo "Running in non-admin mode - admin setup hooks will be skipped"
fi
npm run test:cypress:e2e
- name: Upload test results
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: e2e-results-${{ env.TAG_DIR }}
path: |
packages/cypress/results/
packages/cypress/videos/
packages/cypress/screenshots/
retention-days: 7
- name: Log test completion
if: always()
run: |
echo "🏁 E2E Test completed!"
echo "Status: ${{ job.status }}"
echo "Test Tag: $MATRIX_TAG"
echo "Cluster: $CLUSTER_NAME"
echo "Run ID: ${{ github.run_id }}"
# ---------------------------------------------------------------------------
# Final Status - Update PR with test results
# ---------------------------------------------------------------------------
set-final-status:
needs: [select-cluster, e2e-tests]
if: >-
always() &&
(github.event_name == 'workflow_dispatch' ||
(github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success'))
runs-on: ubuntu-latest
steps:
- name: Set final status
env:
GH_TOKEN: ${{ github.token }}
run: |
E2E_RESULT="${{ needs.e2e-tests.result }}"
CLUSTER_RESULT="${{ needs.select-cluster.result }}"
CLUSTER="${{ needs.select-cluster.outputs.cluster_name }}"
echo "📊 Job results: select-cluster=$CLUSTER_RESULT, e2e-tests=$E2E_RESULT"
# Handle cluster selection failure first
if [[ "$CLUSTER_RESULT" == "failure" ]]; then
STATE="failure"
DESC="Cluster health check failed - no healthy cluster available"
elif [[ "$E2E_RESULT" == "success" ]]; then
STATE="success"
DESC="All tests passed on $CLUSTER"
elif [[ "$E2E_RESULT" == "cancelled" ]]; then
STATE="error"
DESC="Tests cancelled"
elif [[ "$E2E_RESULT" == "skipped" && "$CLUSTER_RESULT" == "skipped" ]]; then
# Both skipped means test.yml failed - don't post status
echo "Both jobs skipped (test.yml likely failed) - not posting status"
exit 0
elif [[ "$E2E_RESULT" == "skipped" ]]; then
STATE="failure"
DESC="Tests skipped due to upstream failure"
else
STATE="failure"
DESC="Tests failed on ${CLUSTER:-unknown cluster}"
fi
echo "📝 Posting status: state=$STATE, description=$DESC"
gh api repos/${{ github.repository }}/statuses/${{ github.event.workflow_run.head_sha || github.sha }} \
-f state="$STATE" \
-f context="Cypress E2E Tests" \
-f description="$DESC" \
-f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
# ---------------------------------------------------------------------------
# Cleanup - Fix file permissions left by BFF test binaries
# ---------------------------------------------------------------------------
cleanup:
needs: [e2e-tests]
runs-on: self-hosted
if: ${{ always() && (github.event_name == 'workflow_dispatch' || (github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success')) }}
steps:
- name: Fix envtest binary permissions
run: |
chmod -R u+w ${{ github.workspace }}/packages/*/bff/bin ${{ github.workspace }}/packages/*/upstream/bff/bin 2>/dev/null || true