From 27e41e82899626cbab4086cf04de1d7a34e24ef0 Mon Sep 17 00:00:00 2001
From: lkstrp <lkstrp@pm.me>
Date: Tue, 3 Feb 2026 14:35:55 +0100
Subject: [PATCH] ci: split integration tests in multiple runners

---
 .github/workflows/test.yaml | 64 ++++++++++++++++++++++++++++---------
 1 file changed, 49 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 85ed088bd4..3131ad7116 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -36,18 +36,57 @@ jobs:
         cache: true
         cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
 
+    - name: Run pylint check on scripts
+      if: (steps.filter.outputs.src == 'true' || github.event_name == 'schedule') && matrix.test == 'electricity'
+      run: |
+        pixi run pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*
+
     - name: Run unit tests
       run: |
         pixi run unit-tests
 
   integration-tests:
-    name: Integration
+    name: Integration - ${{ matrix.test }} (${{ matrix.os }})
     runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
       matrix:
-        # Run windows only on scheduled runs on Sundays, otherwise ignore
         os: ${{ github.event.schedule == '0 5 * * 0' && fromJson('["ubuntu", "macos", "windows"]') || fromJson('["ubuntu", "macos"]') }}
+        test: [electricity, scenarios, overnight, myopic, clusters, tyndp]
+        exclude:
+        - os: macos
+          test: scenarios
+        - os: macos
+          test: overnight
+        - os: macos
+          test: myopic
+        - os: macos
+          test: clusters
+        - os: macos
+          test: tyndp
+        - os: windows
+          test: scenarios
+        - os: windows
+          test: overnight
+        - os: windows
+          test: myopic
+        - os: windows
+          test: clusters
+        - os: windows
+          test: tyndp
+        include:
+        - test: electricity
+          snakemake_cmd: pixi run snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml
+        - test: scenarios
+          snakemake_cmd: pixi run snakemake -call --configfile config/test/config.scenarios.yaml -n
+        - test: overnight
+          snakemake_cmd: pixi run snakemake -call --configfile config/test/config.overnight.yaml
+        - test: myopic
+          snakemake_cmd: pixi run snakemake -call --configfile config/test/config.myopic.yaml
+        - test: clusters
+          snakemake_cmd: pixi run snakemake -call resources/test-elec-clusters/networks/base_s_adm.nc --configfile config/test/config.clusters.yaml
+        - test: tyndp
+          snakemake_cmd: pixi run snakemake -call plot_power_networks_clustered --configfile config/test/config.tyndp.yaml
 
     defaults:
       run:
@@ -70,7 +109,12 @@ jobs:
             - 'pixi.lock'
             - '.github/workflows/test.yaml'
 
+    - name: Skip - no source changes
+      if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule'
+      run: echo "Skipping tests because no source code changes detected"
+
     - name: Free up disk space
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: |
         echo "Initial disk space"
         df -h
@@ -81,10 +125,6 @@ jobs:
         echo "Final disk space"
         df -h
 
-    - name: Skip - no source changes
-      if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule'
-      run: echo "Skipping tests because no source code changes detected"
-
     - name: Setup Pixi
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       uses: prefix-dev/setup-pixi@v0.9.4
@@ -116,25 +156,19 @@ jobs:
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: git checkout HEAD -- data/
 
-    - name: Run pylint check on scripts
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
-      # check for undefined variables to reuse functions across scripts
-      run: |
-        pixi run pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*
-
     - name: Run snakemake test workflows
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
-        pixi run integration-tests
+        ${{ matrix.snakemake_cmd }}
 
     - name: Upload artifacts
       if: always()
       uses: actions/upload-artifact@v6
       with:
-        name: results-${{ matrix.os }}
+        name: results-${{ matrix.test }}-${{ matrix.os }}
         path: |
           logs
           .snakemake/log
@@ -142,5 +176,5 @@ jobs:
         retention-days: 3
 
     - name: Show remaining disk space
-      if: always()
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: df -h