test_with_tox.yml

---
# Runs pytest in several environments including several Python and dependencies versions as well as on different systems.
# All environments are defined in `tox.ini <https://github.com/nilearn/nilearn/blob/main/tox.ini>`_.
#
# .. admonition:: Control via commit message
#    :class: tip
#
#    This workflow can be controlled if your commit message contains:
#
#    - ``[skip test]`` skip this workflow.
#
# Uploads as artifacts:
#
# - ``report.html`` that gives a browsable overview of the tests results
#
# - ``pytest_output`` that contains a .csv of the tests results and figures showing the run time of all the tests
#   See ``maint_tools/plot_test_timing.py``.
#
# - ``htmlcov`` containing the html version of the code coverage.
#
# .. admonition:: Not all python version run on pull requests.
#    :class: warning
#
#    When this workflow is triggered by a pull-request,
#    it will run using only the oldest and latest python version we support.
#    When running on main all combinations of operating system
#    and python versions are run.
###
name: test

on:
    push:
        branches:
        -   main
        paths:
        -   nilearn/**/*
        -   tox.ini
        -   pyproject.toml
        -   .github/workflows/test_with_tox.yml
    pull_request:
        branches:
        -   '*'
        paths:
        -   nilearn/**/*
        -   tox.ini
        -   pyproject.toml
        -   .github/workflows/test_with_tox.yml

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

# Force to use color
env:
    FORCE_COLOR: true

jobs:

    check_skip_flags:
        uses: ./.github/workflows/check_skip_flags.yml
        with:
            skip_type: test

    define-matrix:
        runs-on: ubuntu-latest
        needs: check_skip_flags
        outputs:
            py: ${{ steps.py.outputs.py }}

        steps:
        -   name: Define python version to test
            # only run latest and oldest python versions on PR
            id: py
            run: |
                versions='py=["3.14", "3.13", "3.12", "3.11", "3.10"]'

                if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
                    versions='py=["3.14", "3.10"]'
                fi

                echo "Will run on the following python: ${versions}"
                echo ${versions} >> "$GITHUB_OUTPUT"

    test_and_coverage:
        needs: [check_skip_flags, define-matrix]

        if: github.repository == 'nilearn/nilearn'

        name: 'Test with ${{ matrix.py }} on ${{ matrix.os }}: ${{ matrix.description }}'

        runs-on: ${{ matrix.os }}

        strategy:
            fail-fast: true
            matrix:
                description: [latest dependencies]
                py: ${{ fromJSON(needs.define-matrix.outputs.py) }}
                os: [windows-latest, macos-latest, ubuntu-latest]
                env: [plotting]
                include:
                # Using macos on the following as tests run faster on macos.
                # As 'no plotting' run fewer tests, this allows to fail event faster if using macos.
                -   description: oldest dependencies - no plotting
                    py: '3.10'
                    os: macos-latest
                    env: min
                -   description: oldest dependencies - with plotting
                    py: '3.10'
                    os: macos-latest
                    env: plot_min
                # plotly images saved on different OS can generate slightly
                # different baseline images. As baseline images are generated
                # on a linux machine, ubuntu-latest is used
                -   description: check generated figures
                    py: '3.10'
                    os: ubuntu-latest
                    env: pytest_mpl

        steps:
        -   uses: actions/checkout@v6
            with:
                # If pull request, checkout HEAD commit with all commit history
                ref: ${{ github.event.pull_request.head.sha }}
                fetch-depth: 0

        -   name: Install the latest version of uv
            uses: astral-sh/setup-uv@v7

        -   name: Setup python
            uses: actions/setup-python@v6
            with:
                python-version: ${{ matrix.py }}
                allow-prereleases: true

        -   name: Install tox
            run: uv tool install tox --with=tox-uv --with=tox-gh-actions

        -   name: Show tox config
            run: tox c

        -   name: Merge with upstream
            # We do this so that when on a PR,
            # we only get the files involved in a PR
            # to avoid running more tests than needed.
            shell: bash -el {0}
            run: ./build_tools/github/merge_upstream.sh

        -   name: Run test suite
            shell: bash -el {0}
            run: |
                python build_tools/github/restrict_tests_to_run.py
                tests_to_run=$(cat tests_to_run.txt)
                tox run --list-dependencies -e ${{ matrix.env }} -- ${tests_to_run}
                tox run -e plot_test_timing

        -   name: Upload test report
            if: success() || failure()
            uses: actions/upload-artifact@v7
            with:
                name: ${{ matrix.os }}_${{ matrix.py }}_${{ matrix.description }}_report
                path: |
                    report.html
                    report_slow.html
                    report_doc.html

        -   name: Upload test timings
            if: success() || failure()
            uses: actions/upload-artifact@v7
            with:
                name: ${{ matrix.os }}_${{ matrix.py }}_${{ matrix.description }}_pytest_output
                path: results/pytest_output

        -   name: Upload html coverage
            if: success() || failure()
            uses: actions/upload-artifact@v7
            with:
                name: ${{ matrix.os }}_${{ matrix.py }}_${{ matrix.description }}_htmlcov
                path: htmlcov

        -   name: Upload coverage to CodeCov
            if: success()
            uses: codecov/codecov-action@v5
            with:
                flags: ${{ matrix.os }}_${{ matrix.py }}_${{ matrix.env }}
                token: ${{ secrets.CODECOV_TOKEN }}

    test_pre_release:
        # Test with eventual pre-release version of the dependencies
        # Run separately to avoid blocking other jobs in case of failure
        # (as pre-release dependencies stability is not under our control).
        needs: [check_skip_flags]

        if: github.repository == 'nilearn/nilearn'

        name: 'Test with ${{ matrix.py }} on ${{ matrix.os }}: ${{ matrix.description }}'

        runs-on: ${{ matrix.os }}

        strategy:
            fail-fast: false
            matrix:
                description: [pre-release dependencies]
                py: ['3.14']
                os: [ubuntu-latest]
                env: [pre]

        steps:
        -   uses: actions/checkout@v6
            with:
                # If pull request, checkout HEAD commit with all commit history
                ref: ${{ github.event.pull_request.head.sha }}
                fetch-depth: 0

        -   name: Install the latest version of uv
            uses: astral-sh/setup-uv@v7

        -   name: Setup python
            uses: actions/setup-python@v6
            with:
                python-version: ${{ matrix.py }}
                allow-prereleases: true

        -   name: Install tox
            run: uv tool install tox --with=tox-uv --with=tox-gh-actions

        -   name: Show tox config
            run: tox c

        -   name: Merge with upstream
            # We do this so that when on a PR,
            # we only get the files involved in a PR
            # to avoid running more tests than needed.
            shell: bash -el {0}
            run: ./build_tools/github/merge_upstream.sh

        -   name: Run test suite
            shell: bash -el {0}
            run: |
                python build_tools/github/restrict_tests_to_run.py
                tests_to_run=$(cat tests_to_run.txt)
                tox run --list-dependencies -e ${{ matrix.env }} -- ${tests_to_run}
                tox run -e plot_test_timing

        -   name: Upload test report
            if: success() || failure()
            uses: actions/upload-artifact@v7
            with:
                name: ${{ matrix.os }}_${{ matrix.py }}_${{ matrix.description }}_report
                path: |
                    report.html
                    report_slow.html
                    report_doc.html

        -   name: Upload test timings
            if: success() || failure()
            uses: actions/upload-artifact@v7
            with:
                name: ${{ matrix.os }}_${{ matrix.py }}_${{ matrix.description }}_pytest_output
                path: results/pytest_output

    test_flaky:
        # Some tests are marked as flaky in order to reduce the noise in CI results.
        # These tests are run separately here to monitor their status.
        # We also run them on a single processor to reduce the chance of intermittent failures.
        needs: [check_skip_flags, define-matrix]

        if: github.repository == 'nilearn/nilearn'

        name: 'Test with ${{ matrix.py }} on ${{ matrix.os }}: ${{ matrix.description }}'

        runs-on: ${{ matrix.os }}

        strategy:
            fail-fast: false
            matrix:
                description: [flaky tests]
                py: ${{ fromJSON(needs.define-matrix.outputs.py) }}
                os: [ubuntu-latest, macos-latest, windows-latest]
                env: [flaky]

        steps:
        -   uses: actions/checkout@v6
            with:
                # If pull request, checkout HEAD commit with all commit history
                ref: ${{ github.event.pull_request.head.sha }}
                fetch-depth: 0

        -   name: Install the latest version of uv
            uses: astral-sh/setup-uv@v7

        -   name: Setup python
            uses: actions/setup-python@v6
            with:
                python-version: ${{ matrix.py }}
                allow-prereleases: true

        -   name: Install tox
            run: uv tool install tox --with=tox-uv --with=tox-gh-actions

        -   name: Show tox config
            run: tox c

        -   name: Run test suite
            shell: bash -el {0}
            run: |
                tox run --list-dependencies -e ${{ matrix.env }} -- nilearn