diff --git a/.github/workflows/archive_historic_releases.yml b/.github/workflows/archive_historic_releases.yml new file mode 100644 index 0000000..b0887a1 --- /dev/null +++ b/.github/workflows/archive_historic_releases.yml @@ -0,0 +1,51 @@ +name: Archive Historic Releases + +on: + schedule: + # Run nightly at 2 AM UTC + - cron: "0 2 * * *" + workflow_dispatch: # Allow manual triggering + +jobs: + archive: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install dependencies + run: npm ci + + - name: Collect historic release data + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + node scripts/collect-historic.js --days 14 --start-from-yesterday + + - name: Checkout historic-releases branch + run: | + git fetch origin + git checkout -B historic-releases origin/historic-releases || git checkout -b historic-releases + + - name: Commit and push historic data + run: | + git config user.email "release_historian@gardenlinux.io" + git config user.name "release_historian" + + if [ -n "$(git status --porcelain historic/)" ]; then + git add historic/ + git commit -m "Archive historic release data for last 14 days (starting from yesterday)" + git push origin historic-releases + else + echo "No changes to commit" + fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06ca8ed..4d12622 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,7 @@ jobs: - name: Build dashboard run: | npm run packages + npm run historic npm run build env: NODE_ENV: production diff --git a/.gitignore b/.gitignore index eab26c3..5160696 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ _site +.cache .venv shell.nix vendor @@ -15,3 +16,6 @@ dist/ # packages are in packages branch packages/ + +# historic releases are in historic-releases branch +historic/ diff --git a/index.html b/index.html index a132308..0bbb632 100644 --- a/index.html +++ b/index.html @@ -55,6 +55,12 @@

GitHub Authentication

Historic Daily Releases

+ + Navigate to a specific Garden Linux version to view its + daily release status, workflow runs, and package build + information. Enter a version number below and click "Go + to Version" to view that release. +
- +
@@ -85,6 +97,68 @@

Historic Daily Releases

+

Historic Releases Settings

+
+
+ +
+ + +
+
+ Default: 14 days +
+
+ + Set how many historic daily releases to display in the + historic releases section. +
+ The value will be saved as a URL parameter and can be + shared. + +
+ +
+ +

Cache Settings

+
+
+ + + When enabled, the dashboard ignores cached historic + data and always fetches it directly from GitHub. + This can be slower and may hit API rate limits. + +
+
+ +
+

Workflow Branch Settings

@@ -163,26 +237,64 @@

šŸš€ Current Daily Release

class="current-stages" title="Stages: Package | Repo | Build | Publish" > - - + +
+
- + + +
+
- + + +
+
+ > + + +
0) { + console.log(` šŸ“¦ Loaded ${loadedCount} API responses from cache`); + } +} + +// Save API cache entry to filesystem +function saveApiCacheEntry(url, cacheData) { + try { + initCacheDir(); + const filename = urlToCacheFilename(url); + const filePath = join(CACHE_DIR, filename); + const cacheEntry = { + url, // Store URL for reconstruction on load + data: cacheData.data, + status: cacheData.status, + statusText: cacheData.statusText, + headers: cacheData.headers, + cachedAt: new Date().toISOString(), + }; + writeFileSync(filePath, JSON.stringify(cacheEntry, null, 2)); + } catch (error) { + // Don't fail if cache write fails, just log warning + console.warn(` āš ļø Failed to save API cache entry: ${error.message}`); + } +} + +// Print help information and exit +function printHelp() { + console.log(` +Historic Release Data Collector + +Collects historic release data for Garden Linux dashboard caching. +References packages/{glDays}.json instead of duplicating package data. +Archives for yesterday when run in GitHub Actions context. + +USAGE: + node scripts/collect-historic.js [OPTIONS] + +OPTIONS: + --help, -h, -? Show this help message and exit + + --days Number of historic releases to collect (default: 14) + Collects N releases going backwards from the start point + + --gl Collect specific GL version only (overrides other options) + When specified, only this single version is collected + + --start-from-gl Start collection from specific GL version and collect + N days backwards (overrides --start-from-yesterday/today) + Example: --start-from-gl 2000 --days 7 collects GL 2000-1994 + + --output-dir Output directory for JSON files (default: "historic") + Files are saved as {glDays}.json in this directory + + --batch-size Number of GL days to process in parallel (default: 3) + Workflows within each day are processed sequentially + Higher values = faster but more API rate limit pressure + + --start-from-yesterday Start from yesterday (default behavior) + Today's data is still being generated, so yesterday is + the most recent complete day + + --start-from-today Start from today instead of yesterday + Note: Today's data may still be incomplete + + --force Force re-collection even if output files already exist + By default, existing files are skipped to avoid duplicate + API calls. Use this flag to re-fetch and overwrite. + +EXAMPLES: + # Collect last 14 days (default) + node scripts/collect-historic.js + + # Collect last 7 days + node scripts/collect-historic.js --days 7 + + # Collect from specific GL version backwards + node scripts/collect-historic.js --start-from-gl 2000 --days 7 + + # Collect single specific GL version + node scripts/collect-historic.js --gl 2000 + + # Collect with custom output directory + node scripts/collect-historic.js --output-dir custom-historic --days 30 + + # Collect with higher parallelism (faster but more rate limit pressure) + node scripts/collect-historic.js --batch-size 5 --days 14 + +NOTES: + - Requires GITHUB_TOKEN environment variable for authenticated API access + Without token, API calls are rate-limited to 60 requests/hour + With token, rate limit is 5000 requests/hour + + - The script automatically handles rate limiting and will wait for reset + if the rate limit is exceeded + + - Workflows are processed sequentially within each GL day to avoid + overwhelming the API, but multiple GL days can be processed in parallel + + - Package data is read from packages/{glDays}.json files (not duplicated) + + - Output files follow the schema version defined in constants.js + + - Caching: By default, the script skips GL days that already have output files + in the output directory. Use --force to re-collect everything. +`); + process.exit(0); +} + +// Parse command line arguments +function parseArgs() { + const args = process.argv.slice(2); + + // Check for help flags first + if (args.includes("--help") || args.includes("-h") || args.includes("-?")) { + printHelp(); + } + + const config = { + days: 14, + gl: null, + startFromGl: null, + outputDir: "historic", + startFromYesterday: true, // Default to yesterday (today's data is still being generated) + batchSize: 3, // Default batch size for parallel processing of GL days (workflows are processed sequentially) + force: false, // Force re-collection even if files exist + }; + + for (let i = 0; i < args.length; i++) { + if (args[i] === "--days" && i + 1 < args.length) { + config.days = parseInt(args[i + 1], 10); + if (isNaN(config.days) || config.days < 1) { + console.error("Error: --days must be a positive integer"); + process.exit(1); + } + i++; + } else if (args[i] === "--gl" && i + 1 < args.length) { + config.gl = parseInt(args[i + 1], 10); + if (isNaN(config.gl) || config.gl < 1) { + console.error("Error: --gl must be a positive integer"); + process.exit(1); + } + i++; + } else if (args[i] === "--start-from-gl" && i + 1 < args.length) { + config.startFromGl = parseInt(args[i + 1], 10); + if (isNaN(config.startFromGl) || config.startFromGl < 1) { + console.error( + "Error: --start-from-gl must be a positive integer" + ); + process.exit(1); + } + i++; + } else if (args[i] === "--output-dir" && i + 1 < args.length) { + config.outputDir = args[i + 1]; + i++; + } else if (args[i] === "--batch-size" && i + 1 < args.length) { + config.batchSize = parseInt(args[i + 1], 10); + if (isNaN(config.batchSize) || config.batchSize < 1) { + console.error("Error: --batch-size must be a positive integer"); + process.exit(1); + } + i++; + } else if (args[i] === "--start-from-yesterday") { + config.startFromYesterday = true; + } else if (args[i] === "--start-from-today") { + config.startFromYesterday = false; + } else if (args[i] === "--force") { + config.force = true; + } else if ( + args[i] !== "--help" && + args[i] !== "-h" && + args[i] !== "-?" + ) { + console.error(`Error: Unknown option: ${args[i]}`); + console.error("Use --help to see available options"); + process.exit(1); + } + } + + return config; +} + +// getAuthHeaders is now imported from utils-node.js as getAuthHeadersNode + +/** + * Reconstructs a Response object from cached data + * @param {Object} cacheData - Cached response data + * @returns {Response} Response object that can be used with .json() + */ +function reconstructResponseFromCache(cacheData) { + return new Response(JSON.stringify(cacheData.data), { + status: cacheData.status, + statusText: cacheData.statusText, + headers: cacheData.headers, + }); +} + +// Fetch with rate limit handling and retry, with caching +async function fetchWithRetry(url, options, maxRetries = 3) { + // Check cache first + const cacheKey = url; // Use URL as key (options are usually the same for same URL) + if (apiCache.has(cacheKey)) { + const cached = apiCache.get(cacheKey); + return reconstructResponseFromCache(cached); + } + + for (let attempt = 0; attempt < maxRetries; attempt++) { + let response; + try { + response = await fetch(url, options); + } catch (error) { + // Network error (not HTTP error) + if (attempt < maxRetries - 1) { + const backoffTime = Math.pow(2, attempt) * 1000; // 1s, 2s, 4s + console.warn( + ` āš ļø Network error: ${error.message}. Retrying in ${backoffTime / 1000}s... (attempt ${attempt + 1}/${maxRetries})` + ); + await new Promise((resolve) => + setTimeout(resolve, backoffTime) + ); + continue; + } + // Re-throw on final attempt + throw error; + } + + // Check rate limit headers + const remaining = parseInt( + response.headers.get("x-ratelimit-remaining") || "0", + 10 + ); + const resetTime = parseInt( + response.headers.get("x-ratelimit-reset") || "0", + 10 + ); + + if (response.status === 403 && remaining === 0 && resetTime > 0) { + // Rate limited - coordinate waiting across parallel requests + const now = Math.floor(Date.now() / 1000); + const waitTime = resetTime - now + 1; // Add 1 second buffer + + if (waitTime > 0 && waitTime < 3600) { + // Only wait if reasonable (less than 1 hour) + // Use shared state to ensure only one wait happens + const currentTime = Math.floor(Date.now() / 1000); + + // Check if we need to wait and if someone else is already waiting + if (resetTime > rateLimitState.resetTime) { + rateLimitState.resetTime = resetTime; + rateLimitState.waiting = true; + + console.warn( + ` ā³ Rate limit exceeded. Waiting ${waitTime}s until reset (${new Date(resetTime * 1000).toLocaleTimeString()})...` + ); + + // Create a shared wait promise + rateLimitState.waitPromise = new Promise((resolve) => + setTimeout(resolve, waitTime * 1000) + ); + + await rateLimitState.waitPromise; + rateLimitState.waiting = false; + rateLimitState.waitPromise = null; + } else if ( + rateLimitState.waiting && + rateLimitState.waitPromise + ) { + // Another request is already waiting, join that wait + await rateLimitState.waitPromise; + } + + continue; // Retry after waiting + } + } + + if (response.status === 403 && attempt < maxRetries - 1) { + // Exponential backoff for 403 errors (when not rate limited) + const backoffTime = Math.pow(2, attempt) * 1000; // 1s, 2s, 4s + console.warn( + ` ā³ Rate limited (403). Retrying in ${backoffTime / 1000}s... (attempt ${attempt + 1}/${maxRetries})` + ); + await new Promise((resolve) => setTimeout(resolve, backoffTime)); + continue; + } + + // Cache successful responses before returning + if (response.ok) { + try { + // Clone response to read body without consuming original + const clonedResponse = response.clone(); + const data = await clonedResponse.json(); + const headers = {}; + response.headers.forEach((value, key) => { + headers[key] = value; + }); + const cacheData = { + data, + status: response.status, + statusText: response.statusText, + headers, + }; + apiCache.set(cacheKey, cacheData); + // Persist to filesystem + saveApiCacheEntry(cacheKey, cacheData); + } catch (jsonError) { + // If JSON parsing fails, continue without caching + } + } + + return response; + } + + // Final attempt + const response = await fetch(url, options); + // Cache successful final attempt + if (response.ok) { + try { + // Clone response to read body without consuming original + const clonedResponse = response.clone(); + const data = await clonedResponse.json(); + const headers = {}; + response.headers.forEach((value, key) => { + headers[key] = value; + }); + const cacheData = { + data, + status: response.status, + statusText: response.statusText, + headers, + }; + apiCache.set(cacheKey, cacheData); + // Persist to filesystem + saveApiCacheEntry(cacheKey, cacheData); + } catch (jsonError) { + // If JSON parsing fails, continue without caching + } + } + return response; +} + +// Date calculation functions are now imported from src/utils.js +// getGlDaysFromDate is imported from utils-node.js + +// Get package status summary from packages file +function getPackageStatusSummary(glDays) { + const packagesPath = join(ROOT_DIR, "packages", `${glDays}.json`); + + if (!existsSync(packagesPath)) { + return { + status: "no-data", + issueCount: 0, + totalCount: 0, + }; + } + + try { + const packagesData = JSON.parse(readFileSync(packagesPath, "utf-8")); + const packages = Array.isArray(packagesData) ? packagesData : []; + + let issueCount = 0; + for (const pkg of packages) { + if (PACKAGE_STATUSES.PROBLEMATIC.includes(pkg.Status)) { + issueCount++; + } + } + + return { + status: issueCount > 0 ? "warning" : "success", + issueCount, + totalCount: packages.length, + }; + } catch (error) { + console.warn(`Failed to read packages/${glDays}.json:`, error.message); + return { + status: "error", + issueCount: 0, + totalCount: 0, + }; + } +} + +// collectStage3RunIds is now imported from utils-node.js as collectStage3RunIdsNode + +// Workflow processing functions are now imported from utils-node.js + +// Status calculation functions are now imported from src/utils.js + +// Collect historic data for a single GL day +async function collectHistoricDay(glDays) { + // Skip GL versions older than minimum supported version + if (glDays < MIN_GL_VERSION) { + throw new Error( + `GL${glDays} is older than minimum supported version GL${MIN_GL_VERSION} (workflow structure changed before this version)` + ); + } + + console.log(`\nšŸ“¦ Collecting data for GL${glDays}...`); + + const glDate = formatDetailedDate(glDays); + const { targetDate, nextDay, extendedDate, extendedNextDay } = + calculateDateRanges(glDays, GL_INITIAL_DATE); + + console.log( + ` šŸ“… Date: ${glDate} (${targetDate.toISOString().split("T")[0]})` + ); + + // Get package status summary (references packages file, doesn't duplicate) + console.log(` šŸ“¦ Loading package status from packages/${glDays}.json...`); + const packageStatus = getPackageStatusSummary(glDays); + console.log( + ` šŸ“¦ Package status: ${packageStatus.status} (${packageStatus.issueCount} issues, ${packageStatus.totalCount} total)` + ); + + // Collect Stage 3 run IDs + console.log(` šŸ” Collecting Stage 3 run IDs...`); + const stage3Workflows = [ + { + id: WORKFLOW_IDS.NIGHTLY, + repo: WORKFLOWS.NIGHTLY.repo, + name: WORKFLOWS.NIGHTLY.name, + }, + { + id: WORKFLOW_IDS.MANUAL_RELEASE, + repo: WORKFLOWS.MANUAL_RELEASE.repo, + name: WORKFLOWS.MANUAL_RELEASE.name, + }, + ]; + const stage3RunIds = await collectStage3RunIdsNode( + stage3Workflows, + targetDate, + nextDay, + glDays, + fetchWithRetry, + getAuthHeadersNode + ); + console.log(` šŸ” Found ${stage3RunIds.size} Stage 3 run IDs`); + + // Collect workflow data using shared utility + const workflowChecks = getAllWorkflowChecks(); + + console.log( + ` šŸ”„ Collecting workflow data for ${workflowChecks.length} workflows sequentially...` + ); + const workflowStatuses = {}; + const workflowRunData = {}; + const workflowRuns = {}; + const workflowMetadata = {}; + + // Process workflows sequentially (no parallelization at workflow level) + for (const workflow of workflowChecks) { + const startTime = Date.now(); + try { + console.log( + ` šŸ”„ Processing ${workflow.name} (${workflow.id})...` + ); + // Use pagination to fetch all runs + const runs = await fetchWorkflowRunsPaginatedNode( + workflow, + targetDate, + nextDay, + fetchWithRetry, + getAuthHeadersNode + ); + const fetchTime = Date.now() - startTime; + console.log( + ` šŸ“Š ${workflow.name}: Found ${runs.length} runs (${fetchTime}ms)` + ); + + const result = await processWorkflowRunsNode( + workflow, + runs, + targetDate, + nextDay, + extendedNextDay, + stage3RunIds, + glDays, + fetchWithRetry, + getAuthHeadersNode + ); + + const totalTime = Date.now() - startTime; + console.log( + ` āœ“ ${workflow.name}: ${result.status}${result.runData ? ` (run ${result.runData.id})` : ""} (${totalTime}ms)` + ); + + workflowStatuses[workflow.id] = result.status; + if (result.runData) { + workflowRunData[workflow.id] = result.runData; + } + if (result.allRuns && result.allRuns.length > 0) { + workflowRuns[workflow.id] = { + repo: workflow.repo, + workflowFile: workflow.workflowFile, + workflowName: workflow.name, + runs: result.allRuns.map((run) => ({ + id: run.id, + html_url: run.html_url, + status: run.status, + conclusion: run.conclusion, + created_at: run.created_at, + updated_at: run.updated_at, + head_branch: run.head_branch, + head_sha: run.head_sha, + workflow_url: + run.workflow_url || + `https://github.com/${GARDENLINUX_ORG}/${workflow.repo}/actions/workflows/${workflow.workflowFile}`, + })), + }; + } + + workflowMetadata[workflow.id] = { + repo: workflow.repo, + workflowFile: workflow.workflowFile, + name: workflow.name, + workflowUrl: `https://github.com/${GARDENLINUX_ORG}/${workflow.repo}/actions/workflows/${workflow.workflowFile}`, + }; + } catch (error) { + const totalTime = Date.now() - startTime; + console.warn( + ` āš ļø Error processing ${workflow.name} (${totalTime}ms):`, + error.message + ); + workflowStatuses[workflow.id] = "unknown"; + } + } + + // Calculate stage and pipeline statuses + console.log(` šŸ“Š Calculating stage and pipeline statuses...`); + const stageStatuses = calculateStageStatuses( + workflowStatuses, + packageStatus.status, + STAGE_WORKFLOWS + ); + const pipelineStatus = calculatePipelineStatus(stageStatuses); + const duration = calculateHistoricPipelineDuration( + workflowRunData, + WORKFLOW_IDS + ); + console.log( + ` šŸ“Š Pipeline status: ${pipelineStatus}${duration ? ` (duration: ${duration})` : ""}` + ); + + // Build output structure + const output = { + schemaVersion: HISTORIC_CACHE_SCHEMA_VERSION, + glDays, + date: glDate, + timestamp: new Date().toISOString(), + cached: true, + + // Package data reference (not duplicated) + packageDataPath: `packages/${glDays}.json`, + packageIssuesPath: `packages/${glDays}.json`, + packageStatus, + + // Individual workflow statuses + workflowStatuses, + + // Aggregated stage statuses + workflowStatus: stageStatuses, + + // Pipeline status + pipelineStatus, + duration, + + // Workflow run data + workflowRuns, + + // Workflow metadata + workflowMetadata, + }; + + return output; +} + +// Main function +async function main() { + const config = parseArgs(); + + console.log("šŸš€ Starting historic release data collection"); + console.log(`šŸ“‹ Configuration:`); + console.log(` - Days: ${config.days}`); + console.log(` - Output directory: ${config.outputDir}`); + console.log(` - Batch size: ${config.batchSize}`); + if (config.gl) { + console.log(` - Specific GL version: ${config.gl}`); + } else if (config.startFromGl) { + console.log(` - Start from GL version: ${config.startFromGl}`); + } else { + console.log( + ` - Start from: ${config.startFromYesterday ? "yesterday" : "today"}` + ); + } + + // Determine GL days to collect + let glDaysList = []; + + if (config.gl) { + // Priority 1: --gl flag (collects single version, overrides everything) + glDaysList = [config.gl]; + console.log(`šŸ“… Collecting single GL version: GL${config.gl}`); + } else if (config.startFromGl) { + // Priority 2: --start-from-gl flag (starts from specific GL, collects N days backwards) + console.log( + `šŸ“… Starting from GL${config.startFromGl}, collecting ${config.days} days backwards` + ); + for (let i = 0; i < config.days; i++) { + glDaysList.push(config.startFromGl - i); + } + } else { + // Priority 3: Default behavior (yesterday/today) + const today = new Date(); + today.setHours(0, 0, 0, 0); + + let startGlDays; + if (config.startFromYesterday) { + const yesterday = new Date(today); + yesterday.setDate(yesterday.getDate() - 1); + startGlDays = getGlDaysFromDate(yesterday); + console.log(`šŸ“… Starting from yesterday (GL${startGlDays})`); + } else { + startGlDays = getGlDaysFromDate(today); + console.log(`šŸ“… Starting from today (GL${startGlDays})`); + } + + for (let i = 0; i < config.days; i++) { + glDaysList.push(startGlDays - i); + } + } + + // Filter out GL versions older than minimum supported version + const originalCount = glDaysList.length; + glDaysList = glDaysList.filter((glDays) => glDays >= MIN_GL_VERSION); + const filteredCount = originalCount - glDaysList.length; + + if (filteredCount > 0) { + console.warn( + `āš ļø Skipping ${filteredCount} GL version(s) older than GL${MIN_GL_VERSION} (workflow structure changed before this version)` + ); + } + + if (glDaysList.length === 0) { + console.error( + `āŒ No GL versions to collect (all were filtered out as older than GL${MIN_GL_VERSION})` + ); + process.exit(1); + } + + console.log( + `šŸ“‹ Will collect data for ${glDaysList.length} GL version(s): ${glDaysList.join(", ")}` + ); + + // Load API cache from filesystem + console.log("šŸ“¦ Loading API cache from filesystem..."); + loadApiCache(); + + // Create output directory + const outputDir = join(ROOT_DIR, config.outputDir); + if (!existsSync(outputDir)) { + console.log(`šŸ“ Creating output directory: ${outputDir}`); + mkdirSync(outputDir, { recursive: true }); + } else { + console.log(`šŸ“ Using output directory: ${outputDir}`); + } + + // Collect data for each GL day in parallel batches + let successCount = 0; + let failCount = 0; + let cachedCount = 0; + + // Process GL days in batches for parallel processing + const glDaysBatchSize = config.batchSize; // Use same batch size for GL days + for (let i = 0; i < glDaysList.length; i += glDaysBatchSize) { + const glDaysBatch = glDaysList.slice(i, i + glDaysBatchSize); + const batchNum = Math.floor(i / glDaysBatchSize) + 1; + const totalBatches = Math.ceil(glDaysList.length / glDaysBatchSize); + + console.log( + `\nšŸ“¦ Processing GL days batch ${batchNum}/${totalBatches} (${glDaysBatch.length} releases in parallel)...` + ); + const batchStartTime = Date.now(); + + const glDaysPromises = glDaysBatch.map(async (glDays) => { + try { + const outputPath = join(outputDir, `${glDays}.json`); + + // Check if file already exists and we're not forcing + if (!config.force && existsSync(outputPath)) { + try { + const cachedData = JSON.parse( + readFileSync(outputPath, "utf-8") + ); + console.log( + ` āŠ™ Using cached data for GL${glDays} (use --force to re-collect)` + ); + return { success: true, glDays, cached: true }; + } catch (cacheError) { + console.warn( + ` āš ļø Failed to read cache for GL${glDays}, re-collecting:`, + cacheError.message + ); + // Fall through to re-collect + } + } + + // Collect data (will use API cache for duplicate requests) + const data = await collectHistoricDay(glDays); + writeFileSync(outputPath, JSON.stringify(data, null, 2)); + console.log(` āœ“ Saved ${outputPath}`); + return { success: true, glDays, cached: false }; + } catch (error) { + console.error( + ` āœ— Failed to collect GL${glDays}:`, + error.message + ); + return { success: false, glDays, error: error.message }; + } + }); + + const batchResults = await Promise.all(glDaysPromises); + const batchTime = Date.now() - batchStartTime; + + for (const result of batchResults) { + if (result.success) { + successCount++; + if (result.cached) { + cachedCount++; + } + } else { + failCount++; + } + } + + console.log( + `āœ… GL days batch ${batchNum}/${totalBatches} completed in ${batchTime}ms` + ); + + // Small delay between GL day batches to avoid rate limiting (except for last batch) + if (i + glDaysBatchSize < glDaysList.length) { + await new Promise((resolve) => setTimeout(resolve, 200)); // 200ms delay + } + } + + console.log(`\nāœ… Collection complete!`); + console.log(` - Successfully collected: ${successCount}`); + if (cachedCount > 0) { + console.log(` - Used cached data: ${cachedCount}`); + } + if (failCount > 0) { + console.log(` - Failed: ${failCount}`); + } + if (apiCache.size > 0) { + console.log( + ` - API requests cached: ${apiCache.size} (avoided duplicate requests)` + ); + } +} + +main().catch((error) => { + console.error("Fatal error:", error); + process.exit(1); +}); diff --git a/scripts/utils-node.js b/scripts/utils-node.js new file mode 100644 index 0000000..0d815f1 --- /dev/null +++ b/scripts/utils-node.js @@ -0,0 +1,423 @@ +#!/usr/bin/env node +/** + * Node.js-specific utilities for collect-historic.js + * + * Provides Node.js-compatible versions of browser utilities from src/utils.js + * and src/parentWorkflow.js + */ + +import { + GL_INITIAL_DATE, + API_CONFIG, + WORKFLOW_IDS, + ALLOWED_ARTIFACT_NAMES, +} from "../src/constants.js"; +import { + calculateExpectedBranch, + isBranchMatch, + validateStage4RunCore, +} from "../src/utils.js"; + +/** + * Get GitHub authentication headers for Node.js environment + * Reads token from process.env.GITHUB_TOKEN instead of localStorage + */ +export function getAuthHeadersNode() { + const token = process.env.GITHUB_TOKEN; + if (!token) { + console.warn( + "Warning: GITHUB_TOKEN not set. API calls may be rate-limited." + ); + return { + Accept: "application/vnd.github.v3+json", + "X-GitHub-Api-Version": "2022-11-28", + }; + } + + let authHeader; + if (token.startsWith("ghp_")) { + authHeader = `token ${token}`; + } else if (token.startsWith("github_pat_")) { + authHeader = `Bearer ${token}`; + } else { + authHeader = `token ${token}`; + } + + return { + Authorization: authHeader, + Accept: "application/vnd.github.v3+json", + "X-GitHub-Api-Version": "2022-11-28", + }; +} + +/** + * Calculate GL days from a Date object + */ +export function getGlDaysFromDate(date) { + const initialDay = new Date(GL_INITIAL_DATE); + const targetTime = date.getTime(); + const initialTime = initialDay.getTime(); + return Math.round((targetTime - initialTime) / (1000 * 60 * 60 * 24)); +} + +/** + * Fetches workflow runs with pagination using GitHub's created date filter (Node.js version) + * @param {Object} workflow - Workflow configuration object + * @param {Date} targetDate - Target date to search for + * @param {Date} nextDay - Day after target date (exclusive boundary) + * @param {Function} fetchWithRetry - Function to fetch with retry logic + * @param {Function} getAuthHeadersNode - Function to get auth headers + * @returns {Promise} Array of all collected workflow runs + */ +export async function fetchWorkflowRunsPaginatedNode( + workflow, + targetDate, + nextDay, + fetchWithRetry, + getAuthHeadersNode +) { + const allRuns = []; + const perPage = 100; + + // Format dates for GitHub API (YYYY-MM-DD format, UTC) + // GitHub's date range is inclusive on both ends + // To get runs from targetDate (inclusive) to nextDay (exclusive): + // - fromDate: targetDate (inclusive start) + // - toDate: nextDay - 1 day (inclusive end, covers the full day of targetDate) + const fromDate = targetDate.toISOString().split("T")[0]; + const nextDayStr = nextDay.toISOString().split("T")[0]; + + // Use nextDay as the end date (inclusive), which effectively makes it exclusive + // because we filter client-side with < nextDay + const createdParam = `created=${fromDate}..${nextDayStr}`; + + // Use repository-level endpoint which supports 'created' parameter + // Then filter by workflow_id client-side + // According to GitHub API docs, workflow-specific endpoint may not support 'created' + try { + // Fetch first page to get total_count using repository-level endpoint + const firstPageUrl = `${API_CONFIG.GITHUB_API_BASE}/repos/${API_CONFIG.GARDENLINUX_ORG}/${workflow.repo}/actions/runs?per_page=${perPage}&page=1&${createdParam}`; + + const firstResponse = await fetchWithRetry(firstPageUrl, { + headers: getAuthHeadersNode(), + }); + + if (!firstResponse.ok) { + console.warn( + ` āš ļø [Pagination] Failed to fetch workflow runs for ${workflow.name}: ${firstResponse.status}` + ); + return []; + } + + const firstData = await firstResponse.json(); + const firstPageRuns = firstData.workflow_runs || []; + + // Filter by workflow_id and date + const filteredRuns = firstPageRuns.filter((run) => { + // Match workflow by workflow_id (can be string or number) + const runWorkflowId = run.workflow_id?.toString(); + const targetWorkflowId = workflow.id.toString(); + if (runWorkflowId !== targetWorkflowId) return false; + + // Filter by date + if (!run.created_at) return false; + const runDate = new Date(run.created_at); + return runDate >= targetDate && runDate < nextDay; + }); + + allRuns.push(...filteredRuns); + + const totalCount = firstData.total_count || 0; + const maxResults = 1000; // GitHub API limit for filtered results + const maxPages = Math.min( + Math.ceil(totalCount / perPage), + Math.ceil(maxResults / perPage) + ); // Cap at 10 pages (1000 results) + + // Warn if results may be truncated + if (totalCount >= maxResults) { + console.warn( + ` āš ļø [Pagination] ${workflow.name}: Found ${totalCount} runs in date range (API limit: ${maxResults}). Results may be truncated.` + ); + } + + // Fetch remaining pages if needed + for (let page = 2; page <= maxPages; page++) { + const pageUrl = `${API_CONFIG.GITHUB_API_BASE}/repos/${API_CONFIG.GARDENLINUX_ORG}/${workflow.repo}/actions/runs?per_page=${perPage}&page=${page}&${createdParam}`; + + const pageResponse = await fetchWithRetry(pageUrl, { + headers: getAuthHeadersNode(), + }); + + if (!pageResponse.ok) { + console.warn( + ` āš ļø [Pagination] Failed to fetch page ${page} for ${workflow.name}: ${pageResponse.status}` + ); + break; + } + + const pageData = await pageResponse.json(); + const pageRuns = pageData.workflow_runs || []; + + if (pageRuns.length === 0) { + break; + } + + // Filter by workflow_id and date + const pageFilteredRuns = pageRuns.filter((run) => { + const runWorkflowId = run.workflow_id?.toString(); + const targetWorkflowId = workflow.id.toString(); + if (runWorkflowId !== targetWorkflowId) return false; + + if (!run.created_at) return false; + const runDate = new Date(run.created_at); + return runDate >= targetDate && runDate < nextDay; + }); + + allRuns.push(...pageFilteredRuns); + } + + const pagesFetched = Math.min( + maxPages, + Math.ceil(totalCount / perPage) + ); + console.log( + ` šŸ“„ [Pagination] ${workflow.name}: Fetched ${pagesFetched} page${pagesFetched !== 1 ? "s" : ""} (${allRuns.length} runs for workflow ${workflow.id} in date range ${fromDate}..${nextDayStr})` + ); + + return allRuns; + } catch (error) { + console.warn( + ` āš ļø [Pagination] Error fetching workflow runs for ${workflow.name}:`, + error.message + ); + return []; + } +} + +/** + * Collect Stage 3 run IDs for a specific GL date (Node.js version) + * Uses fetchWithRetry for rate limit handling + */ +export async function collectStage3RunIdsNode( + stage3Workflows, + targetDate, + nextDay, + glDays, + fetchWithRetry, + getAuthHeadersNode +) { + const stage3RunIds = new Set(); + + for (const workflow of stage3Workflows) { + try { + console.log( + ` šŸ” Fetching Stage 3 runs for ${workflow.name}...` + ); + // Use pagination to fetch all runs + const runs = await fetchWorkflowRunsPaginatedNode( + workflow, + targetDate, + nextDay, + fetchWithRetry, + getAuthHeadersNode + ); + + // Runs are already filtered by date in fetchWorkflowRunsPaginatedNode + const dayRuns = runs; + + console.log( + ` šŸ“Š Found ${dayRuns.length} Stage 3 runs for ${workflow.name} in date range` + ); + + for (const run of dayRuns) { + stage3RunIds.add(String(run.id)); + } + } catch (error) { + console.warn( + ` āš ļø Error collecting Stage 3 runs for ${workflow.name}:`, + error.message + ); + } + } + + return stage3RunIds; +} + +/** + * Get parent workflow info (Node.js version) + * Simplified version that checks run event instead of downloading artifacts + */ +export async function getParentWorkflowInfoNode( + owner, + repo, + runId, + fetchWithRetry, + getAuthHeadersNode +) { + try { + const url = `${API_CONFIG.GITHUB_API_BASE}/repos/${owner}/${repo}/actions/runs/${runId}`; + const response = await fetchWithRetry(url, { + headers: getAuthHeadersNode(), + }); + + if (!response.ok) { + return null; + } + + const run = await response.json(); + + // Check for parent workflow run ID in workflow_run event + return { + parentRunId: + run.event === "workflow_run" ? run.workflow_run?.id : null, + }; + } catch (error) { + return null; + } +} + +/** + * Validate Stage 4 runs (Node.js version) + */ +export async function validateStage4RunsNode( + runs, + targetDate, + nextDay, + extendedNextDay, + stage3RunIds, + workflow, + fetchWithRetry, + getAuthHeadersNode, + glDays +) { + // Calculate expected branch using shared utility + const expectedBranch = calculateExpectedBranch(workflow, glDays); + + const validRuns = []; + + for (const run of runs) { + // Filter by branch first using shared utility + const runBranch = run.head_branch || "main"; + const isCorrectBranch = isBranchMatch( + runBranch, + expectedBranch, + workflow, + glDays + ); + + if (!isCorrectBranch) { + console.log( + ` [Branch Filter] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${run.id}: Excluded (branch "${runBranch}" doesn't match expected branch)` + ); + continue; + } + + // Use shared core validation logic (no logging for Node.js version) + const getParentInfoFn = async () => { + return await getParentWorkflowInfoNode( + API_CONFIG.GARDENLINUX_ORG, + workflow.repo, + run.id, + fetchWithRetry, + getAuthHeadersNode + ); + }; + + const isValid = await validateStage4RunCore( + run, + targetDate, + nextDay, + extendedNextDay, + stage3RunIds, + getParentInfoFn, + null // No logging for Node.js version + ); + + if (isValid) { + validRuns.push(run); + } + } + + return validRuns.sort( + (a, b) => new Date(b.created_at) - new Date(a.created_at) + ); +} + +/** + * Process workflow runs (Node.js version) + */ +export async function processWorkflowRunsNode( + workflow, + runs, + targetDate, + nextDay, + extendedNextDay, + stage3RunIds, + glDays, + fetchWithRetry, + getAuthHeadersNode +) { + const isStage4Workflow = workflow.stage === "stage-4"; + + // Calculate expected branch using shared utility + const expectedBranch = calculateExpectedBranch(workflow, glDays); + + let targetRuns = []; + if (isStage4Workflow) { + targetRuns = await validateStage4RunsNode( + runs, + targetDate, + nextDay, + extendedNextDay, + stage3RunIds, + workflow, + fetchWithRetry, + getAuthHeadersNode, + glDays + ); + } else { + targetRuns = runs.filter((run) => { + const runDate = new Date(run.created_at); + const isInDateRange = runDate >= targetDate && runDate < nextDay; + + // Filter by branch using shared utility + const runBranch = run.head_branch || "main"; + const isCorrectBranch = isBranchMatch( + runBranch, + expectedBranch, + workflow, + glDays + ); + + if (!isCorrectBranch && isInDateRange) { + console.log( + ` [Branch Filter] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${run.id}: Excluded (branch "${runBranch}" doesn't match expected branch)` + ); + } + + return isInDateRange && isCorrectBranch; + }); + } + + if (targetRuns.length === 0) { + return { status: "unknown", runData: null, allRuns: [] }; + } + + const sortedRuns = targetRuns.sort( + (a, b) => new Date(b.created_at) - new Date(a.created_at) + ); + const mostRecentRun = sortedRuns[0]; + + let status = "unknown"; + if ( + mostRecentRun.status === "in_progress" || + mostRecentRun.status === "queued" + ) { + status = "progress"; + } else if (mostRecentRun.status === "completed") { + status = mostRecentRun.conclusion === "success" ? "success" : "failure"; + } + + return { status, runData: mostRecentRun, allRuns: sortedRuns }; +} diff --git a/src/constants.js b/src/constants.js index 9f2baf7..4230f33 100644 --- a/src/constants.js +++ b/src/constants.js @@ -18,6 +18,7 @@ // DATE & VERSION CONFIGURATION // ======================================== export const GL_INITIAL_DATE = "2020-03-31"; +export const MIN_GL_VERSION = 1825; // Earliest GL version to retrieve data from (workflow structure changed before this) // ======================================== // ARTIFACT CONFIGURATION @@ -159,3 +160,9 @@ export const UI_CONFIG = { BATCH_SIZE: 3, // API request batch size for rate limiting BATCH_DELAY: 200, // Delay between batches in milliseconds }; + +// ======================================== +// HISTORIC CACHE SCHEMA VERSIONING +// ======================================== +export const HISTORIC_CACHE_SCHEMA_VERSION = 1; +export const HISTORIC_CACHE_MIN_SUPPORTED_VERSION = 1; diff --git a/src/dashboard.js b/src/dashboard.js index ff83987..afd89cb 100644 --- a/src/dashboard.js +++ b/src/dashboard.js @@ -21,6 +21,7 @@ import { getAuthHeaders, isHistoricView, getGlDays, + getCurrentGlDays, formatGLDate, formatDetailedDate, shouldLoadHistoricReleases, @@ -34,10 +35,16 @@ import { calculatePipelineStatus, processWorkflowRuns, getRepoBranchParameter, + getHistoricReleasesCount, + loadHistoricFromCache, + fetchWorkflowRunsPaginated, + calculateDateRanges, + getAllWorkflowChecks, } from "./utils.js"; import { GL_INITIAL_DATE, + MIN_GL_VERSION, WORKFLOWS, WORKFLOW_IDS, STAGE_WORKFLOWS, @@ -135,19 +142,108 @@ export async function getRun() { // Calculate the target date based on GL version const glDays = getGlDays(); - const initialDay = new Date(GL_INITIAL_DATE); - const targetDate = new Date(initialDay); - targetDate.setDate(targetDate.getDate() + glDays); - targetDate.setHours(0, 0, 0, 0); + const currentGlDays = getCurrentGlDays(); - const nextDay = new Date(targetDate); - nextDay.setDate(nextDay.getDate() + 1); + // Check if GL version is too old (workflow structure changed before MIN_GL_VERSION) + if (glDays < MIN_GL_VERSION) { + console.warn( + `[Dashboard] GL${glDays} is older than minimum supported version GL${MIN_GL_VERSION}. Workflow structure changed before this version.` + ); + // Show error state in UI + workflowStatuses = {}; + workflowRunData = {}; + updatePipelineHierarchy(); + return; + } + + // Check if we're viewing a historic GL version (not today's GL) + const isHistoricGl = glDays !== currentGlDays; + + // Try to load from cache first if viewing historic GL + if (isHistoricGl) { + const cachedData = await loadHistoricFromCache(glDays); + if (cachedData) { + console.log( + `[Dashboard] Using cached data for current release view GL${glDays}` + ); + + // Transform cached data to populate workflowStatuses and workflowRunData + // workflowStatuses is already in the right format + workflowStatuses = cachedData.workflowStatuses + ? { ...cachedData.workflowStatuses } + : {}; + + // Transform workflowRuns to workflowRunData + // workflowRuns structure: { [workflowId]: { runs: [...], ... } } + // workflowRunData expects: { [workflowId]: mostRecentRun } + if (cachedData.workflowRuns) { + for (const [workflowId, workflowRunInfo] of Object.entries( + cachedData.workflowRuns + )) { + if ( + workflowRunInfo.runs && + workflowRunInfo.runs.length > 0 + ) { + // Runs are already sorted (newest first), so take the first one + workflowRunData[workflowId] = workflowRunInfo.runs[0]; + } + } + } + + // Get package status for summary + const packageStatus = cachedData.packageStatus || { + status: "unknown", + issueCount: 0, + totalCount: 0, + }; + + // Calculate stage statuses from workflow statuses + const stageStatuses = calculateStageStatuses( + workflowStatuses, + packageStatus.status, + STAGE_WORKFLOWS + ); + + // Update UI with cached data + updatePipelineHierarchy(); + updateCurrentReleaseSummary( + stageStatuses, + cachedData.pipelineStatus, + packageStatus, + workflowRunData, + WORKFLOW_IDS, + () => glDays, + workflowStatuses + ); - // For Stage 4, extend date range to GL + 7 days - const extendedDate = new Date(targetDate); - extendedDate.setDate(extendedDate.getDate() + 7); - const extendedNextDay = new Date(extendedDate); - extendedNextDay.setDate(extendedNextDay.getDate() + 1); + // Render workflow details from cached runs + console.log( + `[Dashboard] Rendering workflow details from cache for GL${glDays}`, + { + workflowRunsKeys: cachedData.workflowRuns + ? Object.keys(cachedData.workflowRuns) + : [], + workflowMetadataKeys: cachedData.workflowMetadata + ? Object.keys(cachedData.workflowMetadata) + : [], + } + ); + await renderWorkflowDetailsFromCache( + cachedData.workflowRuns, + cachedData.workflowMetadata + ); + + return; // Exit early, don't make API calls + } else { + console.log( + `[Dashboard] Cache miss for current release view GL${glDays}, fetching from API` + ); + } + } + + // Fall back to API-based approach (for current GL or cache miss) + const { targetDate, nextDay, extendedDate, extendedNextDay } = + calculateDateRanges(glDays, GL_INITIAL_DATE); // Collect Stage 3 run IDs for parent matching in Stage 4 const stage3RunIds = new Set(); @@ -905,6 +1001,181 @@ async function processWorkflow( workflowDomElement.appendChild(detailsDiv); } +/** + * Renders workflow details from cached data + * @param {Object} workflowRuns - Cached workflow runs data + * @param {Object} workflowMetadata - Cached workflow metadata + */ +async function renderWorkflowDetailsFromCache(workflowRuns, workflowMetadata) { + console.log(`[Cache Render] Starting renderWorkflowDetailsFromCache`, { + hasWorkflowRuns: !!workflowRuns, + hasWorkflowMetadata: !!workflowMetadata, + workflowRunsKeys: workflowRuns ? Object.keys(workflowRuns) : [], + }); + + const allWorkflows = getAllWorkflowConfigs(); + + for (const workflow of allWorkflows) { + const workflowId = workflow.id; + const workflowRunInfo = workflowRuns?.[workflowId]; + const metadata = workflowMetadata?.[workflowId]; + + console.log( + `[Cache Render] Processing workflow ${workflowId} (${workflow.name})`, + { + hasRunInfo: !!workflowRunInfo, + hasRuns: !!workflowRunInfo?.runs, + runsCount: workflowRunInfo?.runs?.length || 0, + } + ); + + if ( + !workflowRunInfo || + !workflowRunInfo.runs || + workflowRunInfo.runs.length === 0 + ) { + // No runs for this workflow, mark as unknown + const workflowDomElement = document.getElementById( + `daily-info-${workflowId}` + ); + if (workflowDomElement) { + setElementStatus(workflowDomElement, "unknown"); + console.log( + `[Cache Render] No runs for ${workflowId}, marked as unknown` + ); + } else { + console.warn( + `[Cache Render] DOM element not found for workflow ${workflowId}` + ); + } + continue; + } + + // Get the workflow DOM element (same ID format as processWorkflow uses) + const workflowDomElement = document.getElementById( + `daily-info-${workflowId}` + ); + if (!workflowDomElement) { + console.warn( + `[Cache] Workflow DOM element not found: daily-info-${workflowId}` + ); + continue; + } + + // Clear existing content (same as processWorkflow does) + const existingDetails = + workflowDomElement.querySelector(".workflow-details"); + if (existingDetails) { + existingDetails.remove(); + } + + // Reset all status classes + setElementStatus(workflowDomElement, null); // Clear all status classes + + // Get the most recent run (first in sorted array) + const mostRecentRun = workflowRunInfo.runs[0]; + const allRuns = workflowRunInfo.runs; + + // Determine workflow status class from most recent run (same logic as processWorkflow) + const { statusClass } = getRunStatus(mostRecentRun); + const workflowStatusClass = statusClass; + + // Also determine workflow status string for tracking + let workflowStatus = "unknown"; + if ( + mostRecentRun.status === "in_progress" || + mostRecentRun.status === "queued" + ) { + workflowStatus = "progress"; + } else if (mostRecentRun.status === "completed") { + workflowStatus = + mostRecentRun.conclusion === "success" ? "success" : "failure"; + } + + // Track status for color coding (same as processWorkflow) + workflowRunData[workflowId] = mostRecentRun; + workflowStatuses[workflowId] = workflowStatus; + + // Update workflow element status + setElementStatus(workflowDomElement, workflowStatusClass); + + // Handle special cases for subsection headers (same as processWorkflow) + const isCloudCleanup = workflowId === WORKFLOW_IDS.CLOUD_TEST_CLEANUP; + const isSnapshot = workflowId === WORKFLOW_IDS.SNAPSHOT; + const isRepoBuild = workflowId === WORKFLOW_IDS.REPO_BUILD; + const isRepoUpdate = workflowId === WORKFLOW_IDS.REPO_UPDATE; + + if (isCloudCleanup) { + const headerElement = document.getElementById( + "cloud-cleanup-header" + ); + if (headerElement) { + let headerStatus = statusClass; + if (statusClass === "queued") headerStatus = "progress"; + setElementStatus(headerElement, headerStatus, "status-"); + updateWorkflowMonitoringHeader(); + } + } + if (isSnapshot) { + const snapshotHeader = document.getElementById("snapshot-header"); + if (snapshotHeader) { + let headerStatus = statusClass; + if (statusClass === "queued") headerStatus = "progress"; + // Check for override status + const overrideStatus = snapshotHeader.dataset.overrideStatus; + if (overrideStatus) { + headerStatus = overrideStatus; + } + setElementStatus(snapshotHeader, headerStatus, "status-"); + updateWorkflowMonitoringHeader(); + } + } + if (isRepoBuild) { + const repoBuildHeader = document.querySelector( + "#sub-stage-repo-build .sub-stage-header" + ); + if (repoBuildHeader) { + let headerStatus = statusClass; + if (statusClass === "queued") headerStatus = "progress"; + setElementStatus(repoBuildHeader, headerStatus, "status-"); + } + } + if (isRepoUpdate) { + const repoUpdateHeader = document.querySelector( + "#sub-stage-repo-update .sub-stage-header" + ); + if (repoUpdateHeader) { + let headerStatus = statusClass; + if (statusClass === "queued") headerStatus = "progress"; + setElementStatus(repoUpdateHeader, headerStatus, "status-"); + } + } + + // Create details section for all runs + const detailsDiv = document.createElement("div"); + detailsDiv.className = "workflow-details"; + + // Render each run + for (const run of allRuns) { + const runDiv = document.createElement("div"); + runDiv.className = "run-item"; + + // Use metadata if available, otherwise use workflow config + const workflowForRender = metadata || workflow; + + // Use full date for Cloud Test Cleanup, time only for others + runDiv.innerHTML = await createRunItemHTML( + run, + workflowForRender, + isCloudCleanup + ); + detailsDiv.appendChild(runDiv); + } + + workflowDomElement.appendChild(detailsDiv); + } +} + // ======================================== // PACKAGE MANAGEMENT FUNCTIONALITY // ======================================== @@ -1143,7 +1414,8 @@ export function updatePipelineHierarchy() { packageStatus, workflowRunData, WORKFLOW_IDS, - getGlDays + getGlDays, + workflowStatuses ); console.log("Stage statuses:", stageStatuses); @@ -1182,9 +1454,10 @@ export async function loadHistoricReleases() { loadingDiv.style.display = "block"; try { - // Load data for the last 14 days (excluding current day) + // Load data for the specified number of days (excluding current day) + const historicCount = getHistoricReleasesCount(); const historicPromises = []; - for (let i = 1; i <= UI_CONFIG.HISTORIC_RELEASES_COUNT; i++) { + for (let i = 1; i <= historicCount; i++) { const historicGL = baseGL - i; if (historicGL > 0) { historicPromises.push(loadHistoricDay(historicGL)); @@ -1231,6 +1504,42 @@ export async function loadHistoricReleases() { async function loadHistoricDay(glDays) { try { + // Skip GL versions older than minimum supported version + if (glDays < MIN_GL_VERSION) { + console.warn( + `[Dashboard] GL${glDays} is older than minimum supported version GL${MIN_GL_VERSION}. Skipping.` + ); + return null; + } + + // Always try to load from cache first + const cachedData = await loadHistoricFromCache(glDays); + + if (cachedData) { + // Use cached data + console.log(`[Dashboard] Using cached data for GL${glDays}`); + + // Transform cached data to match expected format + return { + glDays: cachedData.glDays, + date: cachedData.date, + packageStatus: cachedData.packageStatus, + workflowStatus: cachedData.workflowStatus, // stage statuses + duration: cachedData.duration, + pipelineStatus: cachedData.pipelineStatus, + cached: true, + // Store full cached data for later use + workflowStatuses: cachedData.workflowStatuses, + workflowRunData: cachedData.workflowRuns, + workflowMetadata: cachedData.workflowMetadata, + }; + } + + // Cache miss - fall back to API calls + console.log( + `[Dashboard] Cache miss for GL${glDays}, fetching from API` + ); + // Get basic info const glDate = formatDetailedDate(glDays); @@ -1261,9 +1570,12 @@ async function loadHistoricDay(glDays) { glDays, date: glDate, packageStatus, - workflowStatus: stageStatuses, // for small dots + workflowStatus: stageStatuses, // for small dots (aggregated stage statuses) + workflowStatuses, // individual workflow statuses for stacked dots duration, pipelineStatus, // for main dot and row coloring + cached: false, + workflowRunData, }; } catch (error) { console.warn( @@ -1324,61 +1636,12 @@ async function getHistoricWorkflowStatuses(glDays) { const workflowStatuses = {}; const workflowRunData = {}; - const targetDate = calculateTargetDate(glDays, GL_INITIAL_DATE); - const nextDay = new Date(targetDate); - nextDay.setDate(nextDay.getDate() + 1); - - // For Stage 4 extended date range: GL day + 7 - const extendedDate = new Date(targetDate); - extendedDate.setDate(extendedDate.getDate() + 7); - const extendedNextDay = new Date(extendedDate); - extendedNextDay.setDate(extendedNextDay.getDate() + 1); + const { targetDate, nextDay, extendedDate, extendedNextDay } = + calculateDateRanges(glDays, GL_INITIAL_DATE); try { - // Check multiple workflows per stage for better coverage using constants - const workflowChecks = [ - // Stage 2: Repository workflows - { - id: WORKFLOW_IDS.REPO_UPDATE, - stage: "stage-2", - repo: WORKFLOWS.REPO_UPDATE.repo, - name: WORKFLOWS.REPO_UPDATE.name, - }, - { - id: WORKFLOW_IDS.REPO_BUILD, - stage: "stage-2", - repo: WORKFLOWS.REPO_BUILD.repo, - name: WORKFLOWS.REPO_BUILD.name, - }, - - // Stage 3: Build & Release workflows - { - id: WORKFLOW_IDS.NIGHTLY, - stage: "stage-3", - repo: WORKFLOWS.NIGHTLY.repo, - name: WORKFLOWS.NIGHTLY.name, - }, - { - id: WORKFLOW_IDS.MANUAL_RELEASE, - stage: "stage-3", - repo: WORKFLOWS.MANUAL_RELEASE.repo, - name: WORKFLOWS.MANUAL_RELEASE.name, - }, - - // Stage 4: Publish workflows - { - id: WORKFLOW_IDS.PUBLISH_GHCR, - stage: "stage-4", - repo: WORKFLOWS.PUBLISH_GHCR.repo, - name: WORKFLOWS.PUBLISH_GHCR.name, - }, - { - id: WORKFLOW_IDS.PUBLISH_S3, - stage: "stage-4", - repo: WORKFLOWS.PUBLISH_S3.repo, - name: WORKFLOWS.PUBLISH_S3.name, - }, - ]; + // Get standard workflow list using shared utility + const workflowChecks = getAllWorkflowChecks(); // Collect Stage 3 run IDs first const stage3Workflows = workflowChecks.filter( @@ -1394,45 +1657,16 @@ async function getHistoricWorkflowStatuses(glDays) { // Process workflows with timeout and better error handling const promises = workflowChecks.map(async (workflow) => { try { - // eslint-disable-next-line no-undef - const controller = new AbortController(); - const timeoutId = setTimeout( - () => controller.abort(), - API_CONFIG.TIMEOUT - ); - - const response = await fetch( - `${API_CONFIG.GITHUB_API_BASE}/repos/${API_CONFIG.GARDENLINUX_ORG}/${workflow.repo}/actions/workflows/${workflow.id}/runs?per_page=${API_CONFIG.HISTORIC_RUNS_PER_PAGE}${workflow.repo === "repo" ? getRepoBranchParameter() : getBranchParameter()}`, - { - headers: getAuthHeaders(), - signal: controller.signal, - } + // Use pagination to fetch all runs + const runs = await fetchWorkflowRunsPaginated( + workflow, + targetDate, + nextDay, + getAuthHeaders, + getBranchParameter, + getRepoBranchParameter ); - clearTimeout(timeoutId); - - if (!response.ok) { - console.warn( - `[Dashboard] Historic API error for ${workflow.name} (${workflow.id}):`, - { - status: response.status, - statusText: response.statusText, - workflowId: workflow.id, - workflowName: workflow.name, - repo: workflow.repo, - glDays, - } - ); - return { - workflow, - status: "unknown", - runData: null, - }; - } - - const data = await response.json(); - const runs = data.workflow_runs || []; - // Use shared workflow processing logic (same as current view) const result = await processWorkflowRuns( workflow, diff --git a/src/main.js b/src/main.js index d868aba..7c60038 100644 --- a/src/main.js +++ b/src/main.js @@ -34,6 +34,7 @@ import { formatDetailedDateFromDate, isHistoricView, getWorkflowsByStage, + getHistoricReleasesCount, } from "./utils.js"; import { generateWorkflowBoxHTML as uiGenerateWorkflowBoxHTML } from "./ui.js"; @@ -62,7 +63,7 @@ window.saveToken = function () { // Validate token format if (!token.startsWith("ghp_") && !token.startsWith("github_pat_")) { console.warn("[Main] Token save attempted with invalid format:", { - tokenPrefix: token.substring(0, 10) + "...", + tokenPrefix: `${token.substring(0, 10)}...`, tokenLength: token.length, }); const confirmSave = confirm( @@ -106,6 +107,56 @@ window.clearToken = function () { } }; +// ======================================== +// HISTORIC RELEASES COUNT SETTINGS +// ======================================== +window.updateHistoricCount = function () { + const input = document.getElementById("historic-count-input"); + if (!input) return; + + const count = parseInt(input.value, 10); + if (isNaN(count) || count < 1 || count > 2000) { + alert("Please enter a valid number between 1 and 2000"); + input.value = getHistoricReleasesCount(); + return; + } + + // Update URL parameter + const url = new URL(window.location); + if (count === 14) { + // Remove parameter if default value + url.searchParams.delete("historic_count"); + } else { + url.searchParams.set("historic_count", count.toString()); + } + + // Navigate to the new URL + window.location.href = url.toString(); +}; + +window.handleHistoricCountKeypress = function (event) { + if (event.key === "Enter") { + window.updateHistoricCount(); + } +}; + +// Set historic count input from URL parameter +function setHistoricCountFromUrl() { + const input = document.getElementById("historic-count-input"); + if (!input) return; + const count = getHistoricReleasesCount(); + input.value = count; + updateHistoricCountInfo(); +} + +// Update historic count info text +function updateHistoricCountInfo() { + const infoElement = document.getElementById("historic-count-info"); + if (!infoElement) return; + const count = getHistoricReleasesCount(); + infoElement.textContent = `Currently showing: ${count} ${count === 1 ? "day" : "days"}`; +} + // ======================================== // BRANCH SEARCH SETTINGS // ======================================== @@ -170,9 +221,15 @@ function setBranchCheckboxFromUrl() { // Call this on DOMContentLoaded or after settings panel is rendered if (document.readyState === "loading") { - document.addEventListener("DOMContentLoaded", setBranchCheckboxFromUrl); + document.addEventListener("DOMContentLoaded", () => { + setBranchCheckboxFromUrl(); + setHistoricCountFromUrl(); + setForceFromUrl(); + }); } else { setBranchCheckboxFromUrl(); + setHistoricCountFromUrl(); + setForceFromUrl(); } function updateAuthStatus() { @@ -233,7 +290,7 @@ window.handleGLKeypress = function (event) { if (event.key === "Enter") { goToGL(); } - // Update date info as user types + // Update date info as user types (no navigation on input) setTimeout(() => { const glInput = document.getElementById("gl-input"); const value = parseInt(glInput.value); @@ -332,6 +389,57 @@ function toggleWorkflowMonitoring() { ); } +// Toggle force cache (disable historic cache usage) +window.toggleForceCache = function () { + try { + const checkbox = document.getElementById("force-cache"); + if (!checkbox) return; + + const url = new URL(window.location); + if (checkbox.checked) { + url.searchParams.set("force", "true"); + } else { + url.searchParams.delete("force"); + } + + window.location.href = url.toString(); + } catch (error) { + console.error("[Main] Error toggling force cache:", { + error: error.message, + stack: error.stack, + }); + } +}; + +// Initialize force cache checkbox from URL parameter +function setForceFromUrl() { + try { + const checkbox = document.getElementById("force-cache"); + if (!checkbox) return; + + const urlParams = new URLSearchParams(window.location.search); + const value = urlParams.get("force"); + if (value === null) { + checkbox.checked = false; + return; + } + + const normalized = value.toLowerCase(); + const isForce = + value === "" || + normalized === "true" || + normalized === "1" || + normalized === "yes"; + + checkbox.checked = isForce; + } catch (error) { + console.error("[Main] Error setting force checkbox from URL:", { + error: error.message, + stack: error.stack, + }); + } +} + // Toggle Debian Snapshot sub-section function toggleSnapshot() { toggleSection("snapshot-content", "snapshot-toggle-icon", getRun); @@ -370,23 +478,27 @@ function generateWorkflowHTML() { .join(""); } + const isHistoric = isHistoricView(); + // Cloud Test Cleanup: render from constants into monitoring sub-stage - const cloudCleanupContainer = document.querySelector( - "#cloud-cleanup-content .cloud-cleanup-workflow" - ); - if (cloudCleanupContainer && WORKFLOWS.CLOUD_TEST_CLEANUP) { - cloudCleanupContainer.innerHTML = uiGenerateWorkflowBoxHTML( - WORKFLOWS.CLOUD_TEST_CLEANUP, - API_CONFIG, - WORKFLOWS + if (!isHistoric) { + const cloudCleanupContainer = document.querySelector( + "#cloud-cleanup-content .cloud-cleanup-workflow" ); + if (cloudCleanupContainer && WORKFLOWS.CLOUD_TEST_CLEANUP) { + cloudCleanupContainer.innerHTML = uiGenerateWorkflowBoxHTML( + WORKFLOWS.CLOUD_TEST_CLEANUP, + API_CONFIG, + WORKFLOWS + ); + } } // Workflow Monitoring: render additional monitoring workflows as sub-stages const monitoringSubsections = document.getElementById( "monitoring-subsections" ); - if (monitoringSubsections && WORKFLOWS.SNAPSHOT) { + if (!isHistoric && monitoringSubsections && WORKFLOWS.SNAPSHOT) { const snapshotSection = document.createElement("div"); snapshotSection.className = "sub-stage"; snapshotSection.id = "sub-stage-snapshot"; @@ -504,7 +616,23 @@ function initDashboard() { ".historic-releases-header h2" ); if (historicReleaseHeader) { - historicReleaseHeader.textContent = `šŸ“… Historic Daily Releases (14 Days Before GL ${glDays})`; + const historicCount = getHistoricReleasesCount(); + historicReleaseHeader.textContent = `šŸ“… Historic Daily Releases (${historicCount} ${historicCount === 1 ? "Day" : "Days"} Before GL ${glDays})`; + } + + // Hide Workflow Monitoring section for historic views + const workflowMonitoringContainer = document.getElementById( + "workflow-monitoring-container" + ); + if (workflowMonitoringContainer) { + workflowMonitoringContainer.style.display = "none"; + console.log( + `[Main] Hidden Workflow Monitoring container for historic GL${glDays}` + ); + } else { + console.warn( + `[Main] Workflow Monitoring container not found when trying to hide for historic GL${glDays}` + ); } } else { glDaysElement.innerText = `GL ${glDays} \n ${formattedDate}`; @@ -530,7 +658,16 @@ function initDashboard() { ".historic-releases-header h2" ); if (historicReleaseHeader) { - historicReleaseHeader.textContent = `šŸ“… Historic Daily Releases (14 Days Before Today)`; + const historicCount = getHistoricReleasesCount(); + historicReleaseHeader.textContent = `šŸ“… Historic Daily Releases (${historicCount} ${historicCount === 1 ? "Day" : "Days"} Before Today)`; + } + + // Show Workflow Monitoring section for current view + const workflowMonitoringContainer = document.getElementById( + "workflow-monitoring-container" + ); + if (workflowMonitoringContainer) { + workflowMonitoringContainer.style.display = ""; } } @@ -567,6 +704,7 @@ function initDashboard() { updateAuthStatus(); // Initialize auth status display initializeGLSelector(); // Initialize GL version selector generateWorkflowHTML(); // Generate workflow HTML + updateHistoricCountInfo(); // Update historic count info display console.log("[Main] Dashboard initialization completed successfully"); } catch (error) { diff --git a/src/ui.js b/src/ui.js index c95267e..c481dbb 100644 --- a/src/ui.js +++ b/src/ui.js @@ -52,8 +52,40 @@ export function renderHistoricReleases(historicData) { })); historicList.innerHTML = safeHistoricData - .map( - (day) => ` + .map((day) => { + // Get individual workflow statuses for stages 2, 3, and 4 + const repoUpdateStatus = + day.workflowStatuses && + day.workflowStatuses[WORKFLOW_IDS.REPO_UPDATE] + ? day.workflowStatuses[WORKFLOW_IDS.REPO_UPDATE] + : "unknown"; + const repoBuildStatus = + day.workflowStatuses && + day.workflowStatuses[WORKFLOW_IDS.REPO_BUILD] + ? day.workflowStatuses[WORKFLOW_IDS.REPO_BUILD] + : "unknown"; + const nightlyStatus = + day.workflowStatuses && + day.workflowStatuses[WORKFLOW_IDS.NIGHTLY] + ? day.workflowStatuses[WORKFLOW_IDS.NIGHTLY] + : "unknown"; + const manualReleaseStatus = + day.workflowStatuses && + day.workflowStatuses[WORKFLOW_IDS.MANUAL_RELEASE] + ? day.workflowStatuses[WORKFLOW_IDS.MANUAL_RELEASE] + : "unknown"; + const publishGhcrStatus = + day.workflowStatuses && + day.workflowStatuses[WORKFLOW_IDS.PUBLISH_GHCR] + ? day.workflowStatuses[WORKFLOW_IDS.PUBLISH_GHCR] + : "unknown"; + const publishS3Status = + day.workflowStatuses && + day.workflowStatuses[WORKFLOW_IDS.PUBLISH_S3] + ? day.workflowStatuses[WORKFLOW_IDS.PUBLISH_S3] + : "unknown"; + + return `
GL ${day.glDays}
${day.date}
@@ -64,10 +96,21 @@ export function renderHistoricReleases(historicData) {
- - - - +
+ +
+
+ + +
+
+ + +
+
+ + +
@@ -104,9 +147,10 @@ export function renderHistoricReleases(historicData) { : "Status loading..." }
+ ${day.cached ? '
cached
' : ""} - ` - ) + `; + }) .join(""); } @@ -136,7 +180,8 @@ export function updateCurrentReleaseSummary( packageStatus, workflowRunData, WORKFLOW_IDS, - getGlDays + getGlDays, + workflowStatuses = {} ) { const glDays = getGlDays(); const formattedDate = formatDetailedDate(glDays); @@ -164,12 +209,53 @@ export function updateCurrentReleaseSummary( } // Update stage dots - for (let i = 1; i <= 4; i++) { - const stageDot = document.getElementById(`current-stage-${i}`); - if (stageDot) { - const stageStatus = stageStatuses[`stage-${i}`] || "unknown"; - setElementStatus(stageDot, stageStatus); - } + // Stage 1: single dot + const stage1Dot = document.getElementById("current-stage-1"); + if (stage1Dot) { + const stageStatus = stageStatuses["stage-1"] || "unknown"; + setElementStatus(stage1Dot, stageStatus); + } + + // Stage 2: stacked dots for individual workflows + const stage2TopDot = document.getElementById("current-stage-2-top"); + const stage2BottomDot = document.getElementById("current-stage-2-bottom"); + if (stage2TopDot) { + const repoUpdateStatus = + workflowStatuses[WORKFLOW_IDS.REPO_UPDATE] || "unknown"; + setElementStatus(stage2TopDot, repoUpdateStatus); + } + if (stage2BottomDot) { + const repoBuildStatus = + workflowStatuses[WORKFLOW_IDS.REPO_BUILD] || "unknown"; + setElementStatus(stage2BottomDot, repoBuildStatus); + } + + // Stage 3: stacked dots for individual workflows + const stage3TopDot = document.getElementById("current-stage-3-top"); + const stage3BottomDot = document.getElementById("current-stage-3-bottom"); + if (stage3TopDot) { + const nightlyStatus = + workflowStatuses[WORKFLOW_IDS.NIGHTLY] || "unknown"; + setElementStatus(stage3TopDot, nightlyStatus); + } + if (stage3BottomDot) { + const manualReleaseStatus = + workflowStatuses[WORKFLOW_IDS.MANUAL_RELEASE] || "unknown"; + setElementStatus(stage3BottomDot, manualReleaseStatus); + } + + // Stage 4: stacked dots for individual workflows + const stage4TopDot = document.getElementById("current-stage-4-top"); + const stage4BottomDot = document.getElementById("current-stage-4-bottom"); + if (stage4TopDot) { + const publishGhcrStatus = + workflowStatuses[WORKFLOW_IDS.PUBLISH_GHCR] || "unknown"; + setElementStatus(stage4TopDot, publishGhcrStatus); + } + if (stage4BottomDot) { + const publishS3Status = + workflowStatuses[WORKFLOW_IDS.PUBLISH_S3] || "unknown"; + setElementStatus(stage4BottomDot, publishS3Status); } // Update summary text diff --git a/src/utils.js b/src/utils.js index d6ff2fd..2755207 100644 --- a/src/utils.js +++ b/src/utils.js @@ -18,7 +18,13 @@ * Core support library used across all dashboard components. */ -import { GL_INITIAL_DATE, WORKFLOWS } from "./constants.js"; +import { + GL_INITIAL_DATE, + WORKFLOWS, + WORKFLOW_IDS, + HISTORIC_CACHE_SCHEMA_VERSION, + HISTORIC_CACHE_MIN_SUPPORTED_VERSION, +} from "./constants.js"; // ======================================== // DATE AND GL VERSION CALCULATIONS @@ -40,6 +46,51 @@ export function getGlDaysFromUrl() { return null; } +export function getHistoricReleasesCount() { + const countParam = getUrlParameter("historic_count"); + if (countParam) { + const count = parseInt(countParam, 10); + if (!isNaN(count) && count > 0 && count <= 2000) { + return count; + } + } + return 14; // Default value +} + +/** + * Checks whether the dashboard should bypass historic cache + * Uses a URL parameter `force`, similar to the CLI flag `--force` + * Examples: + * - ?force -> true + * - ?force=true -> true + * - ?force=1 -> true + * - ?force=false -> false + * - ?force=0 -> false + */ +export function isForceNoCache() { + const urlParams = new URLSearchParams(window.location.search); + if (!urlParams.has("force")) { + return false; + } + + const value = urlParams.get("force"); + // Treat presence, empty, "true" or "1" as enabled by default + if (value === null || value === "") { + return true; + } + + const normalized = value.toLowerCase(); + if (normalized === "true" || normalized === "1") { + return true; + } + if (normalized === "false" || normalized === "0") { + return false; + } + + // Any other non-empty value means force as well + return true; +} + export function getCurrentGlDays() { const today = new Date(); today.setHours(0, 0, 0, 0); @@ -199,6 +250,145 @@ export function shouldSearchAllBranches() { return branchParam === "true" || branchParam === "1"; } +// ======================================== +// BRANCH FILTERING UTILITIES +// ======================================== + +/** + * Calculates the expected branch for a workflow based on its type + * @param {Object} workflow - Workflow configuration object + * @param {number} glDays - GL version days + * @returns {string|null} Expected branch name, or null for Repo Build (special handling) + */ +export function calculateExpectedBranch(workflow, glDays) { + const isRepoWorkflow = workflow.repo === "repo"; + const isRepoUpdate = workflow.id === WORKFLOW_IDS.REPO_UPDATE; + const isRepoBuild = workflow.id === WORKFLOW_IDS.REPO_BUILD; + + if (isRepoWorkflow) { + if (isRepoUpdate) { + // Repo Update always runs on "main" branch + return "main"; + } else if (isRepoBuild) { + // Repo Build runs on version branches in format "{glDays}.0" or "{glDays}.0.0" + // Return null to indicate special handling needed + return null; + } else { + return `${glDays}.0.0`; + } + } else { + // Non-repo workflows run on "main" branch + return "main"; + } +} + +/** + * Checks if a run's branch matches the expected branch for a workflow + * @param {string} runBranch - Branch name from the workflow run + * @param {string|null} expectedBranch - Expected branch (null for Repo Build) + * @param {Object} workflow - Workflow configuration object + * @param {number} glDays - GL version days + * @returns {boolean} True if branch matches + */ +export function isBranchMatch(runBranch, expectedBranch, workflow, glDays) { + const isRepoBuild = workflow.id === WORKFLOW_IDS.REPO_BUILD; + + if (isRepoBuild && expectedBranch === null) { + // Repo Build accepts both "{glDays}.0" and "{glDays}.0.0" formats + return runBranch === `${glDays}.0` || runBranch === `${glDays}.0.0`; + } else { + // Exact match for other workflows + return runBranch === expectedBranch; + } +} + +// ======================================== +// DATE RANGE CALCULATION UTILITIES +// ======================================== + +/** + * Calculates all date ranges needed for GL version processing + * @param {number} glDays - GL version days + * @param {string} initialDate - Initial date string (e.g., "2020-03-31") + * @returns {Object} Object containing targetDate, nextDay, extendedDate, extendedNextDay + */ +export function calculateDateRanges(glDays, initialDate) { + const targetDate = calculateTargetDate(glDays, initialDate); + const nextDay = new Date(targetDate); + nextDay.setDate(nextDay.getDate() + 1); + + // For Stage 4 extended date range: GL day + 7 + const extendedDate = new Date(targetDate); + extendedDate.setDate(extendedDate.getDate() + 7); + const extendedNextDay = new Date(extendedDate); + extendedNextDay.setDate(extendedNextDay.getDate() + 1); + + return { + targetDate, + nextDay, + extendedDate, + extendedNextDay, + }; +} + +// ======================================== +// WORKFLOW LIST UTILITIES +// ======================================== + +/** + * Returns the standard list of workflows to check for historic releases + * @returns {Array} Array of workflow check objects + */ +export function getAllWorkflowChecks() { + return [ + // Stage 2: Repository workflows + { + id: WORKFLOW_IDS.REPO_UPDATE, + stage: "stage-2", + repo: WORKFLOWS.REPO_UPDATE.repo, + name: WORKFLOWS.REPO_UPDATE.name, + workflowFile: WORKFLOWS.REPO_UPDATE.workflowFile, + }, + { + id: WORKFLOW_IDS.REPO_BUILD, + stage: "stage-2", + repo: WORKFLOWS.REPO_BUILD.repo, + name: WORKFLOWS.REPO_BUILD.name, + workflowFile: WORKFLOWS.REPO_BUILD.workflowFile, + }, + // Stage 3: Build & Release workflows + { + id: WORKFLOW_IDS.NIGHTLY, + stage: "stage-3", + repo: WORKFLOWS.NIGHTLY.repo, + name: WORKFLOWS.NIGHTLY.name, + workflowFile: WORKFLOWS.NIGHTLY.workflowFile, + }, + { + id: WORKFLOW_IDS.MANUAL_RELEASE, + stage: "stage-3", + repo: WORKFLOWS.MANUAL_RELEASE.repo, + name: WORKFLOWS.MANUAL_RELEASE.name, + workflowFile: WORKFLOWS.MANUAL_RELEASE.workflowFile, + }, + // Stage 4: Publish workflows + { + id: WORKFLOW_IDS.PUBLISH_GHCR, + stage: "stage-4", + repo: WORKFLOWS.PUBLISH_GHCR.repo, + name: WORKFLOWS.PUBLISH_GHCR.name, + workflowFile: WORKFLOWS.PUBLISH_GHCR.workflowFile, + }, + { + id: WORKFLOW_IDS.PUBLISH_S3, + stage: "stage-4", + repo: WORKFLOWS.PUBLISH_S3.repo, + name: WORKFLOWS.PUBLISH_S3.name, + workflowFile: WORKFLOWS.PUBLISH_S3.workflowFile, + }, + ]; +} + // ======================================== // GITHUB API UTILITIES // ======================================== @@ -583,6 +773,93 @@ export function calculateHistoricPipelineDuration( // STAGE 4 VALIDATION UTILITIES // ======================================== +/** + * Core Stage 4 run validation logic (shared between browser and Node.js) + * @param {Object} run - Workflow run to validate + * @param {Date} targetDate - GL target date + * @param {Date} nextDay - Day after GL target date + * @param {Date} extendedNextDay - GL target date + 7 days + * @param {Set} stage3RunIds - Set of valid Stage 3 run IDs + * @param {Function} getParentInfoFn - Async function to get parent workflow info + * @param {Function|null} logFn - Optional logging function (run, message) => void + * @returns {Promise} True if run is valid + */ +export async function validateStage4RunCore( + run, + targetDate, + nextDay, + extendedNextDay, + stage3RunIds, + getParentInfoFn, + logFn = null +) { + const runDate = new Date(run.created_at); + const isBaseDate = runDate >= targetDate && runDate < nextDay; + const isExtendedDate = runDate >= targetDate && runDate < extendedNextDay; + + try { + const parentInfo = await getParentInfoFn(); + + // Case 1: Same date validation - only valid if no parent info OR parent matches Stage 3 + if (isBaseDate) { + // If there's no parent info, include the run (manual run or no parent data) + if (!parentInfo || !parentInfo.parentRunId) { + if (logFn) { + logFn(run, "Added (GL date, no parent)"); + } + return true; + } + + // If there's a parent ID, it must match a Stage 3 run + if (stage3RunIds.has(parentInfo.parentRunId.toString())) { + if (logFn) { + logFn( + run, + `Added (GL date, matching parent ${parentInfo.parentRunId})` + ); + } + return true; + } + + // Skip runs with parent IDs that don't match Stage 3 + if (logFn) { + logFn( + run, + `Skipped (GL date, parent ${parentInfo.parentRunId} doesn't match Stage 3)` + ); + } + return false; + } + + // Case 2: Later date validation (+1 to +7 days) - only valid if parent run matches Stage 3 + if ( + isExtendedDate && + !isBaseDate && + parentInfo && + parentInfo.parentRunId && + stage3RunIds.has(parentInfo.parentRunId.toString()) + ) { + if (logFn) { + logFn( + run, + `Added (later date, matching parent ${parentInfo.parentRunId})` + ); + } + return true; + } + + if (logFn) { + logFn(run, "Skipped (doesn't match validation criteria)"); + } + return false; + } catch (error) { + if (logFn) { + logFn(run, `Failed to get parent info: ${error.message}`); + } + return false; + } +} + /** * Validates Stage 4 runs based on date and parent criteria * @param {Array} runs - Array of Stage 4 runs to validate @@ -606,76 +883,66 @@ export async function validateStage4Runs( const { getParentWorkflowInfo } = await import("./parentWorkflow.js"); const { API_CONFIG } = await import("./constants.js"); + // Check if user wants to search all branches (respects "Search all branches" setting) + const searchAllBranches = shouldSearchAllBranches(); + + // Calculate expected branch using shared utility + const expectedBranch = calculateExpectedBranch(workflow, glDays); + const validRuns = []; for (const run of runs) { - console.log( - `[DEBUG] [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Pre-filter Run ${run.id}: created_at=${run.created_at}` - ); - - try { - // Get parent workflow info for this run - const parentInfo = await getParentWorkflowInfo( - API_CONFIG.GARDENLINUX_ORG, - workflow.repo, - run.id + // Calculate runBranch once and reuse + const runBranch = run.head_branch || "main"; + + // Filter by branch only if not searching all branches + if (!searchAllBranches) { + const isCorrectBranch = isBranchMatch( + runBranch, + expectedBranch, + workflow, + glDays ); - const runDate = new Date(run.created_at); - const isBaseDate = runDate >= targetDate && runDate < nextDay; - const isExtendedDate = - runDate >= targetDate && runDate < extendedNextDay; - - // Case 1: Same date validation - only valid if no parent info OR parent matches Stage 3 - if (isBaseDate) { - // If there's no parent info, include the run (manual run or no parent data) - if (!parentInfo || !parentInfo.parentRunId) { - validRuns.push(run); - console.log( - `šŸ” [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${run.id}: Added (GL date, no parent)` - ); - continue; - } - - // If there's a parent ID, it must match a Stage 3 run - if (stage3RunIds.has(parentInfo.parentRunId.toString())) { - validRuns.push(run); - console.log( - `šŸ” [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${run.id}: Added (GL date, matching parent ${parentInfo.parentRunId})` - ); - continue; - } - - // Skip runs with parent IDs that don't match Stage 3 + if (!isCorrectBranch) { console.log( - `šŸ” [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${run.id}: Skipped (GL date, parent ${parentInfo.parentRunId} doesn't match Stage 3)` + `[Branch Filter] [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${run.id}: Excluded (branch "${runBranch}" doesn't match expected branch)` ); continue; } + } - // Case 2: Later date validation (+1 to +7 days) - only valid if parent run matches Stage 3 - if ( - isExtendedDate && - !isBaseDate && - parentInfo && - parentInfo.parentRunId && - stage3RunIds.has(parentInfo.parentRunId.toString()) - ) { - validRuns.push(run); - console.log( - `šŸ” [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${run.id}: Added (later date, matching parent ${parentInfo.parentRunId})` - ); - continue; - } + console.log( + `[DEBUG] [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Pre-filter Run ${run.id}: created_at=${run.created_at}, branch=${runBranch}` + ); - console.log( - `šŸ” [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${run.id}: Skipped (doesn't match validation criteria)` + // Use shared core validation logic + const getParentInfoFn = async () => { + return await getParentWorkflowInfo( + API_CONFIG.GARDENLINUX_ORG, + workflow.repo, + run.id ); - } catch (error) { + }; + + const logFn = (runToLog, message) => { console.log( - `šŸ” [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Failed to get parent info for run ${run.id}:`, - error.message + `šŸ” [Historic Stage 4] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${runToLog.id}: ${message}` ); + }; + + const isValid = await validateStage4RunCore( + run, + targetDate, + nextDay, + extendedNextDay, + stage3RunIds, + getParentInfoFn, + logFn + ); + + if (isValid) { + validRuns.push(run); } } @@ -696,6 +963,148 @@ export async function validateStage4Runs( return uniqueRuns; } +/** + * Fetches workflow runs with pagination using GitHub's created date filter + * @param {Object} workflow - Workflow configuration object + * @param {Date} targetDate - Target date to search for + * @param {Date} nextDay - Day after target date (exclusive boundary) + * @param {Function} getAuthHeaders - Function to get auth headers + * @param {Function} getBranchParameter - Function to get branch parameter + * @param {Function} getRepoBranchParameter - Function to get repo branch parameter + * @returns {Promise} Array of all collected workflow runs + */ +export async function fetchWorkflowRunsPaginated( + workflow, + targetDate, + nextDay, + getAuthHeaders, + getBranchParameter, + getRepoBranchParameter +) { + const { API_CONFIG } = await import("./constants.js"); + const allRuns = []; + const perPage = API_CONFIG.HISTORIC_RUNS_PER_PAGE || 100; + + // Format dates for GitHub API (YYYY-MM-DD format, UTC) + // GitHub's date range is inclusive on both ends + // To get runs from targetDate (inclusive) to nextDay (exclusive): + // - fromDate: targetDate (inclusive start) + // - toDate: nextDay (inclusive end, we filter client-side with < nextDay) + const fromDate = targetDate.toISOString().split("T")[0]; + const nextDayStr = nextDay.toISOString().split("T")[0]; + + // Use nextDay as the end date (inclusive), which effectively makes it exclusive + // because we filter client-side with < nextDay + const createdParam = `created=${fromDate}..${nextDayStr}`; + + const branchParam = + workflow.repo === "repo" + ? getRepoBranchParameter() + : getBranchParameter(); + + try { + // Use repository-level endpoint which supports 'created' parameter + // Then filter by workflow_id client-side + // According to GitHub API docs, workflow-specific endpoint may not support 'created' + const firstPageUrl = `${API_CONFIG.GITHUB_API_BASE}/repos/${API_CONFIG.GARDENLINUX_ORG}/${workflow.repo}/actions/runs?per_page=${perPage}&page=1&${createdParam}${branchParam}`; + + const firstResponse = await fetch(firstPageUrl, { + headers: getAuthHeaders(), + }); + + if (!firstResponse.ok) { + console.warn( + `[Pagination] Failed to fetch workflow runs for ${workflow.name}: ${firstResponse.status}` + ); + return []; + } + + const firstData = await firstResponse.json(); + const firstPageRuns = firstData.workflow_runs || []; + + // Filter by workflow_id and date + const filteredRuns = firstPageRuns.filter((run) => { + // Match workflow by workflow_id (can be string or number) + const runWorkflowId = run.workflow_id?.toString(); + const targetWorkflowId = workflow.id.toString(); + if (runWorkflowId !== targetWorkflowId) return false; + + // Filter by date + if (!run.created_at) return false; + const runDate = new Date(run.created_at); + return runDate >= targetDate && runDate < nextDay; + }); + + allRuns.push(...filteredRuns); + + const totalCount = firstData.total_count || 0; + const maxResults = 1000; // GitHub API limit for filtered results + const maxPages = Math.min( + Math.ceil(totalCount / perPage), + Math.ceil(maxResults / perPage) + ); // Cap at 10 pages (1000 results) + + // Warn if results may be truncated + if (totalCount >= maxResults) { + console.warn( + `[Pagination] ${workflow.name}: Found ${totalCount} runs in date range (API limit: ${maxResults}). Results may be truncated.` + ); + } + + // Fetch remaining pages if needed + for (let page = 2; page <= maxPages; page++) { + const pageUrl = `${API_CONFIG.GITHUB_API_BASE}/repos/${API_CONFIG.GARDENLINUX_ORG}/${workflow.repo}/actions/runs?per_page=${perPage}&page=${page}&${createdParam}${branchParam}`; + + const pageResponse = await fetch(pageUrl, { + headers: getAuthHeaders(), + }); + + if (!pageResponse.ok) { + console.warn( + `[Pagination] Failed to fetch page ${page} for ${workflow.name}: ${pageResponse.status}` + ); + break; + } + + const pageData = await pageResponse.json(); + const pageRuns = pageData.workflow_runs || []; + + if (pageRuns.length === 0) { + break; + } + + // Filter by workflow_id and date + const pageFilteredRuns = pageRuns.filter((run) => { + const runWorkflowId = run.workflow_id?.toString(); + const targetWorkflowId = workflow.id.toString(); + if (runWorkflowId !== targetWorkflowId) return false; + + if (!run.created_at) return false; + const runDate = new Date(run.created_at); + return runDate >= targetDate && runDate < nextDay; + }); + + allRuns.push(...pageFilteredRuns); + } + + const pagesFetched = Math.min( + maxPages, + Math.ceil(totalCount / perPage) + ); + console.log( + `šŸ“„ [Pagination] ${workflow.name}: Fetched ${pagesFetched} page${pagesFetched !== 1 ? "s" : ""} (${allRuns.length} runs for workflow ${workflow.id} in date range ${fromDate}..${nextDayStr})` + ); + + return allRuns; + } catch (error) { + console.warn( + `[Pagination] Error fetching workflow runs for ${workflow.name}:`, + error.message + ); + return []; + } +} + /** * Collects Stage 3 run IDs for a specific GL date * @param {Array} stage3Workflows - Array of Stage 3 workflow configurations @@ -713,35 +1122,21 @@ export async function collectStage3RunIds( try { const { getAuthHeaders, getBranchParameter, getRepoBranchParameter } = await import("./utils.js"); - const { API_CONFIG } = await import("./constants.js"); const stage3RunIds = new Set(); for (const workflow of stage3Workflows) { try { - const response = await fetch( - `${API_CONFIG.GITHUB_API_BASE}/repos/${API_CONFIG.GARDENLINUX_ORG}/${workflow.repo}/actions/workflows/${workflow.id}/runs?per_page=${API_CONFIG.HISTORIC_RUNS_PER_PAGE}${workflow.repo === "repo" ? getRepoBranchParameter() : getBranchParameter()}`, - { headers: getAuthHeaders() } + // Use pagination to fetch all runs + const runs = await fetchWorkflowRunsPaginated( + workflow, + targetDate, + nextDay, + getAuthHeaders, + getBranchParameter, + getRepoBranchParameter ); - if (!response.ok) { - console.warn( - `[Utils] Failed to fetch Stage 3 runs for ${workflow.name} (${workflow.id}):`, - { - status: response.status, - statusText: response.statusText, - workflowId: workflow.id, - workflowName: workflow.name, - repo: workflow.repo, - glDays, - } - ); - continue; - } - - const data = await response.json(); - const runs = data.workflow_runs || []; - // Use base date range for Stage 3 run collection (GL date only, not extended) const dayRuns = runs.filter((run) => { const runDate = new Date(run.created_at); @@ -951,6 +1346,12 @@ export async function processWorkflowRuns( // Check if this is a Stage 4 workflow const isStage4Workflow = workflow.stage === "stage-4"; + // Check if user wants to search all branches (respects "Search all branches" setting) + const searchAllBranches = shouldSearchAllBranches(); + + // Calculate expected branch using shared utility + const expectedBranch = calculateExpectedBranch(workflow, glDays); + let targetRuns = []; if (isStage4Workflow) { @@ -968,7 +1369,29 @@ export async function processWorkflowRuns( // Standard date filtering for non-Stage 4 workflows targetRuns = runs.filter((run) => { const runDate = new Date(run.created_at); - return runDate >= targetDate && runDate < nextDay; + const isInDateRange = runDate >= targetDate && runDate < nextDay; + + // Filter by branch only if not searching all branches + if (!searchAllBranches) { + const runBranch = run.head_branch || "main"; + const isCorrectBranch = isBranchMatch( + runBranch, + expectedBranch, + workflow, + glDays + ); + + if (!isCorrectBranch && isInDateRange) { + console.log( + `[Branch Filter] GL${glDays} - ${workflow.name} (${workflow.id}) - Run ${run.id}: Excluded (branch "${runBranch}" doesn't match expected branch)` + ); + } + + return isInDateRange && isCorrectBranch; + } + + // If searching all branches, only filter by date + return isInDateRange; }); } @@ -1044,3 +1467,152 @@ export async function processWorkflowRuns( return { status, runData: mostRecentRun }; } + +// ======================================== +// HISTORIC CACHE UTILITIES +// ======================================== + +/** + * Loads historic release data from cache + * @param {number} glDays - GL version days + * @returns {Promise} Cached historic data or null if unavailable/incompatible + */ +export async function loadHistoricFromCache(glDays) { + // Allow users to bypass cache via force URL parameter + if (isForceNoCache()) { + console.log( + `[Cache] Skipping historic cache for GL${glDays} because force parameter is set` + ); + return null; + } + + try { + console.log( + `[Cache] Attempting to load cache for GL${glDays} from historic/${glDays}.json` + ); + const response = await fetch(`historic/${glDays}.json`); + if (!response.ok) { + console.log( + `[Cache] Cache file not found or not accessible for GL${glDays} (HTTP ${response.status})` + ); + return null; + } + + const data = await response.json(); + console.log( + `[Cache] Successfully loaded cache file for GL${glDays}, validating...` + ); + + // Validate schema version + if (!data.schemaVersion) { + console.warn( + `[Cache] Historic cache for GL${glDays} missing schemaVersion, rejecting` + ); + return null; + } + + const schemaVersion = data.schemaVersion; + + // Check if version is supported + if ( + schemaVersion < HISTORIC_CACHE_MIN_SUPPORTED_VERSION || + schemaVersion > HISTORIC_CACHE_SCHEMA_VERSION + ) { + console.warn( + `[Cache] Historic cache for GL${glDays} has unsupported schema version ${schemaVersion} (supported: ${HISTORIC_CACHE_MIN_SUPPORTED_VERSION}-${HISTORIC_CACHE_SCHEMA_VERSION}), rejecting` + ); + return null; + } + + // Migrate data if needed (for future versions) + const migratedData = migrateHistoricCache(data, schemaVersion); + + // Validate required fields for current schema version + if (!validateHistoricCache(migratedData)) { + console.warn( + `[Cache] Historic cache for GL${glDays} failed validation, rejecting` + ); + return null; + } + + console.log( + `[Cache] Successfully loaded and validated cache for GL${glDays}` + ); + return migratedData; + } catch (error) { + console.warn( + `[Cache] Failed to load historic cache for GL${glDays}:`, + error.message + ); + return null; + } +} + +/** + * Migrates historic cache data to current schema version + * @param {Object} data - Cached data + * @param {number} fromVersion - Source schema version + * @returns {Object} Migrated data + */ +function migrateHistoricCache(data, fromVersion) { + // Currently only version 1 exists, so no migration needed + // Future: Add migration logic here when schema evolves + if (fromVersion === HISTORIC_CACHE_SCHEMA_VERSION) { + return data; + } + + // Placeholder for future migrations + console.warn( + `[Cache] Migration from version ${fromVersion} to ${HISTORIC_CACHE_SCHEMA_VERSION} not implemented` + ); + return data; +} + +/** + * Validates historic cache data structure + * @param {Object} data - Cached data to validate + * @returns {boolean} True if valid + */ +function validateHistoricCache(data) { + // Required fields for schema version 1 + const requiredFields = [ + "schemaVersion", + "glDays", + "date", + "timestamp", + "packageDataPath", + "packageIssuesPath", + "packageStatus", + "workflowStatuses", + "workflowStatus", + "pipelineStatus", + "workflowRuns", + "workflowMetadata", + ]; + + for (const field of requiredFields) { + if (!(field in data)) { + console.warn(`[Cache] Missing required field: ${field}`); + return false; + } + } + + // Validate packageStatus structure + if ( + !data.packageStatus || + typeof data.packageStatus.status !== "string" || + typeof data.packageStatus.issueCount !== "number" || + typeof data.packageStatus.totalCount !== "number" + ) { + console.warn("[Cache] Invalid packageStatus structure"); + return false; + } + + // Validate workflowStatuses is an object + if (!data.workflowStatuses || typeof data.workflowStatuses !== "object") { + console.warn("[Cache] Invalid workflowStatuses structure"); + return false; + } + + return true; +} diff --git a/style.css b/style.css index 8c9af84..3f8217e 100644 --- a/style.css +++ b/style.css @@ -112,6 +112,9 @@ --shadow-md: 0 0.5rem 1rem var(--shadow-color-black-medium); --shadow-lg: 0 1rem 3rem var(--shadow-color-black-large); + /* Stage Dots Spacing */ + --stage-dots-gap: 4px; + /* Text Shadow */ --text-shadow-header: 0 2px 4px var(--shadow-color-black-medium); } @@ -123,8 +126,21 @@ /* Status indicator dots - unified base class (remove current-overall-status to avoid conflicts) */ .status-indicator, .current-status-indicator, +.stage-dot { + display: inline-block; + width: 14px; + height: 14px; + border-radius: 50%; + background-color: var(--color-unknown); + border: 2px solid var(--border-white-transparent); + box-shadow: 0 1px 3px var(--shadow-color-black); + transition: all 0.3s ease; + margin: 0 auto; + flex-shrink: 0; +} + +/* Stage dots (both single and within stacked containers) */ .current-stage-dot, -.stage-dot, .historic-stage-dot { display: inline-block; width: 14px; @@ -134,7 +150,7 @@ border: 2px solid var(--border-white-transparent); box-shadow: 0 1px 3px var(--shadow-color-black); transition: all 0.3s ease; - margin: 0 auto; + margin: 0; flex-shrink: 0; } @@ -956,6 +972,15 @@ table { /* Unified date styling */ .current-date, +.historic-cached-indicator { + display: inline-block; + font-size: 0.7em; + color: var(--color-unknown); + margin-left: 4px; + font-weight: normal; + opacity: 0.7; +} + .historic-date { color: var(--text-unknown); font-size: 1em; @@ -1060,13 +1085,31 @@ table { .current-stages, .historic-stages { display: flex; - gap: 4px; + gap: var(--stage-dots-gap); width: 70px; flex-shrink: 0; align-items: center; justify-content: center; } +/* Single dot container for stage 1 */ +.current-stage-dot-container, +.historic-stage-dot-container { + display: flex; + align-items: center; + justify-content: center; +} + +/* Stacked dots container for stages 2, 3, and 4 */ +.current-stage-dots-stacked, +.historic-stage-dots-stacked { + display: flex; + flex-direction: column; + gap: var(--stage-dots-gap); + align-items: center; + justify-content: center; +} + /* Unified package status styling */ .current-package-status, .historic-package-status { @@ -1099,6 +1142,18 @@ table { font-weight: 600; } +/* Historic cached indicator */ +.historic-cached { + width: 80px; + flex-shrink: 0; + text-align: right; + color: var(--color-unknown); + font-size: 0.9em; + font-weight: 500; + opacity: 0.8; + margin-left: auto; +} + /* PIPELINE HIERARCHY SYSTEM */ .pipeline-container { max-width: 1200px; @@ -1636,6 +1691,14 @@ table { margin-bottom: 15px; } +.gl-description { + display: block; + color: var(--text-unknown); + font-size: 0.8em; + margin-top: 0; + margin-bottom: 12px; +} + .gl-selector label { display: block; margin-bottom: 5px; @@ -1703,6 +1766,9 @@ table { .gl-actions { margin-top: 10px; + display: flex; + flex-direction: column; + gap: 8px; } .today-btn { @@ -1718,6 +1784,7 @@ table { text-decoration: none; display: block; text-align: center; + margin-top: 0; } .today-btn:hover { @@ -1726,6 +1793,59 @@ table { transform: translateY(-1px); } +/* HISTORIC COUNT SETTINGS */ +.historic-count-settings { + margin-bottom: 15px; +} + +.historic-count-group { + margin-bottom: 10px; +} + +.historic-count-group label { + display: block; + margin-bottom: 5px; + font-weight: 500; + color: var(--text-primary); + font-size: 0.9em; +} + +.historic-count-input-group { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 10px; +} + +#historic-count-input { + flex: 1; + padding: 6px 8px; + border: 1px solid var(--border-medium); + border-radius: 4px; + font-size: 0.85em; + font-family: monospace; + background: var(--bg-unknown); + color: var(--text-primary); +} + +#historic-count-input:focus { + outline: none; + border-color: var(--color-info); + box-shadow: 0 0 0 2px var(--shadow-color-info-light); +} + +.historic-count-info { + font-size: 0.8em; + color: var(--text-unknown); +} + +.historic-count-help { + display: block; + color: var(--text-unknown); + font-size: 0.8em; + margin-top: 5px; +} + /* SEQUENTIAL WORKFLOWS & SUB-STAGES */ .sequential-workflows { display: flex;