Apply some important fixes (#579)

- Use `pyccel >= 2.2.3`: this allows compiling all kernels with C
- Require `h5py >= 3.16` to allow for standard install with `setuptools
>= 81.0`
- Require `numpy >= 2.1` to support Python >= 3.10
- Use `pytest.toml` instead of `pytest.ini` for Pytest configuration
(requires `pytest >= 9.0`)
- Move coverage configuration from `pyproject.toml` to
`psydac/pytest.toml`
- Don't run postprocessing unit tests with `pytest-xdist` because `h5py`
is not thread-safe: add Pytest mark `@pytest.mark.xdist_group('h5py')`
to all unit tests in `psydac/api/tests/test_postprocessing.py`
- Return error code on failure of the `psydac test` command; this was
not correctly returned before, hence the CI could not detect when the
unit tests were failing in a subprocess
- Return error code on failure of the `psydac compile` command; this fix
does not directly affect the CI, but it is included for correctness.
- Check correct reporting of failure for `psydac test` command in CI
testing
- Use correct configuration file in coverage CI tests
- For clarity, rename temporary folder where all CI tests are run:
`pytest` -> `scratch`
- Clean up `.gitignore`; specifically, stop ignoring files which don't
exist anymore

Author: yguclu

.github/actions/parallel_h5py/action.yml

@@ -15,32 +15,20 @@ runs:
         echo $HDF5_DIR
         echo "HDF5_DIR=$HDF5_DIR" >> $GITHUB_ENV
 
-# To be deactivated when a new version of h5py is released on PyPI
     - name: Install h5py in parallel mode
       shell: bash
       run: |
         export CC="mpicc"
         export HDF5_MPI="ON"
-        git clone https://github.com/h5py/h5py.git
-        cd h5py
-        pip install -v .
+        pip install h5py --no-cache-dir --no-binary h5py
         pip list
 
-# To be reactivated when a new version of h5py is released on PyPI
-#    - name: Install h5py in parallel mode
-#      shell: bash
-#      run: |
-#        export CC="mpicc"
-#        export HDF5_MPI="ON"
-#        pip install h5py --no-cache-dir --no-binary h5py
-#        pip list
-
     - name: Check parallel h5py installation
       shell: bash
       run: |
-          python -c "
-          from mpi4py import MPI
-          import h5py
-          # This particular instantiation of h5py.File will fail if parallel h5py isn't installed
-          f = h5py.File('parallel_test.hdf5', 'w', driver='mpio', comm=MPI.COMM_WORLD)
-          print(f)"
+        python -c "
+        from mpi4py import MPI
+        import h5py
+        # This particular instantiation of h5py.File will fail if parallel h5py isn't installed
+        f = h5py.File('parallel_test.hdf5', 'w', driver='mpio', comm=MPI.COMM_WORLD)
+        print(f)"

.github/workflows/testing.yml

@@ -8,7 +8,6 @@ on:
     branches: [ devel, main ]
     paths:
       - 'psydac/**'
-      - 'pytest.ini'
       - 'pyproject.toml'
 
   pull_request:
@@ -119,27 +118,37 @@ jobs:
           pip install .[test]
           pip freeze
 
-      - name: Test Pyccel optimization flags
+      - name: Initialize test directory
         run: |
+          mkdir scratch
+
+      - name: Test Pyccel optimization flags
+        working-directory: ./scratch
+        run: >-
+          cp $GITHUB_WORKSPACE/psydac/pytest.toml . &&
           pytest --pyargs psydac -m pyccel --capture=no
 
-      - name: Initialize test directory
+      - name: Verify that 'psydac test' reports failures correctly
+        working-directory: ./scratch
+        # We run a test which is not collected by pytest by default, and we expect it to fail.
+        # If it fails (non-zero), the "echo" runs and the script exits with 0 (success).
+        # If it passes (zero), the "exit 1" runs and the GitHub Action fails.
         run: |
-          mkdir pytest
+          psydac test --mod psydac.cmd.tests.failing_test && { echo "Test passed but should have failed!"; exit 1; } || echo "Test failed as expected."
 
       - name: Run coverage tests on macOS
         if: matrix.os == 'macos-14'
-        working-directory: ./pytest
+        working-directory: ./scratch
         run: >-
+          cp $GITHUB_WORKSPACE/psydac/pytest.toml . &&
           pytest -n auto
           --cov psydac
-          --cov-config $GITHUB_WORKSPACE/pyproject.toml
           --cov-report xml
           --pyargs psydac -m "not mpi and not petsc" -ra
 
       - name: Run single-process tests with Pytest on Ubuntu
         if: matrix.os == 'ubuntu-24.04'
-        working-directory: ./pytest
+        working-directory: ./scratch
         run: |
           psydac test
 
@@ -148,35 +157,38 @@ jobs:
         uses: codacy/codacy-coverage-reporter-action@v1.3.0
         with:
           project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
-          coverage-reports: ./pytest/coverage.xml
+          coverage-reports: ./scratch/coverage.xml
 
       - name: Print detailed coverage results on macOS
         if: matrix.os == 'macos-14'
-        working-directory: ./pytest
+        working-directory: ./scratch
         run: |
           coverage report --ignore-errors --show-missing --sort=cover
 
       - name: Run MPI tests with Pytest
-        working-directory: ./pytest
+        working-directory: ./scratch
         run: |
           psydac test --mpi
 
       - name: Run single-process PETSc tests with Pytest
-        working-directory: ./pytest
+        working-directory: ./scratch
         run: |
           psydac test --petsc
 
       - name: Run MPI PETSc tests with Pytest
-        working-directory: ./pytest
+        working-directory: ./scratch
         run: |
           psydac test --mpi --petsc
 
       - name: Run single-process example tests with Pytest on Ubuntu
         if: matrix.os == 'ubuntu-24.04'
-        run: |
+        working-directory: ./scratch
+        run: >-
+          cp $GITHUB_WORKSPACE/psydac/pytest.toml . &&
+          cp -r $GITHUB_WORKSPACE/examples . &&
           python -m pytest examples/feec
 
       - name: Remove test directory
         if: always()
         run: |
-          rm -rf pytest
+          rm -rf scratch

.gitignore

@@ -11,28 +11,20 @@ build
 *build*
 *egg*
 *dist*
-usr
 *cache*
 
 *.swp
 *.log
 
-doc/_static
-doc/_build
-doc/api-python
-
-psydac/core/bsp-f2py*
-psydac/core/bspmodule.c
-.env
-
 # pytest directories
 __test__/
 
 # pycharm directory
 .idea
 
-# Visual Studio Code workspace files
+# Visual Studio Code
 *.code-workspace
+**/.vscode/
 
 # Meson lock files
 */.wraplock

CHANGELOG.md

@@ -10,15 +10,24 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+-   #579 : Require `h5py>=3.16` which installs correctly with `setuptools>=81.0`
+-   #579 : Don't run postprocessing unit tests with `pytest-xdist` because `h5py` is not thread-safe
+-   #579 : Return error code on failure of the `psydac test` and `psydac compile` commands
 -   #577 : Fix installation following release of Pyccel 2.2
 -   #571 : Fix correct application of the sum factorization algorithm
 -   #570 : Optimize PSYDAC logo
 -   #565 : Expand editable install info in `README.md`
 -   #566 : Fix command `psydac test --mpi` on Ubuntu machines
--   [DEVELOPER] Update CI installation of `h5py` and `petsc4py` after release of `setuptools` 81.0
+-   [DEVELOPER] Update CI installation of `petsc4py` after release of `setuptools` 81.0
+-   [DEVELOPER] Check correct reporting of failure for `psydac test` command in CI testing
+-   [DEVELOPER] Use correct configuration file in coverage CI tests
 
 ### Changed
 
+-   #579 : Require `pyccel>=2.2.3` which can compile all kernels with C
+-   #579 : Require `numpy>=2.1` to support Python >= 3.10
+-   #579 : Require `pytest>=9.0` and use `pytest.toml` instead of `pytest.ini` for Pytest configuration
+-   #579 : Move coverage configuration from `pyproject.toml` to `psydac/pytest.toml`
 -   [DEVELOPER] Do not check file changes to trigger testing workflow on PRs
 -   [DEVELOPER] Run documentation workflow on pushes to `devel` whenever `README.md` is modified
 -   [DEVELOPER] Run testing and documentation workflows on PRs only when set to "ready for review"

README.md

@@ -64,7 +64,7 @@ A developer wanting to modify the latest source code on GitHub should skip that
 git clone --recurse-submodules https://github.com/pyccel/psydac.git
 cd psydac
 
-pip install meson-python "pyccel>=2.2.2"
+pip install meson-python "pyccel>=2.2.3"
 pip install --no-build-isolation --editable ".[test]"
 ```
 

docs/installation.md

@@ -149,7 +149,7 @@ At this point the PSYDAC library may be installed from PyPI in **standard mode**
     git clone --recurse-submodules https://github.com/pyccel/psydac.git
     cd psydac
 
-    pip install meson-python "pyccel>=2.1.0"
+    pip install meson-python "pyccel>=2.2.3"
     pip install --no-build-isolation --editable ".[test]"
     ```
     An equivalent repository address for the `clone` command is `git@github.com:pyccel/psydac.git`, which requires a GitHub account.

psydac/api/tests/test_postprocessing.py

@@ -82,6 +82,7 @@ def build_2_cubes():
 ###############################################################################
 #                            Output Manager tests                             #
 ###############################################################################
+@pytest.mark.xdist_group('h5py')
 @pytest.mark.parametrize( 'dtype', ['float', 'complex'] )
 def test_add_spaces(dtype):
     domain = Square('D')
@@ -206,6 +207,7 @@ def test_add_spaces(dtype):
     os.remove('test_add_spaces_single_patch.yml')
 
 
+@pytest.mark.xdist_group('h5py')
 @pytest.mark.parametrize( 'dtype', ['float', 'complex'] )
 def test_export_fields_serial(dtype):
     domain = Square('D')
@@ -287,6 +289,7 @@ def test_export_fields_serial(dtype):
     os.remove('test_export_fields_serial.h5')
 
 
+@pytest.mark.xdist_group('h5py')
 @pytest.mark.mpi
 def test_export_fields_parallel():
     comm = MPI.COMM_WORLD
@@ -329,6 +332,7 @@ def test_export_fields_parallel():
 ###############################################################################
 #                 Output Manager and PostProcess Manager tests                #
 ###############################################################################
+@pytest.mark.xdist_group('h5py')
 @pytest.mark.parametrize('domain', [Square(), Cube()])
 @pytest.mark.parametrize( 'dtype', ['float', 'complex'] )
 def test_reconstruct_spaces_topological_domain(domain, dtype):
@@ -398,6 +402,7 @@ def test_reconstruct_spaces_topological_domain(domain, dtype):
     os.remove("test_reconstruct_spaces_topological_domain_2.yml")
 
 
+@pytest.mark.xdist_group('h5py')
 @pytest.mark.parametrize('domain, seq', [(Square(), ['h1', 'hdiv', 'l2']), (Square(), ['h1', 'hcurl', 'l2']), (Cube(), None)])
 @pytest.mark.parametrize( 'dtype', ['float', 'complex'] )
 def test_reconstruct_DerhamSequence_topological_domain(domain, seq, dtype):
@@ -456,6 +461,7 @@ def test_reconstruct_DerhamSequence_topological_domain(domain, seq, dtype):
     os.remove('test_reconstruct_DerhamSequence_topological_domain.yml')
 
 
+@pytest.mark.xdist_group('h5py')
 @pytest.mark.parametrize('geometry, seq', [('identity_2d.h5', ['h1', 'hdiv', 'l2']),
                                            ('identity_2d.h5', ['h1', 'hcurl', 'l2']),
                                            ('identity_3d.h5', None),
@@ -521,6 +527,8 @@ def test_reconstruct_DerhamSequence_discrete_domain(geometry, seq, dtype):
     os.remove('test_reconstruct_DerhamSequence_discrete_domain_2.yml')
     os.remove('test_reconstruct_DerhamSequence_discrete_domain.yml')
 
+
+@pytest.mark.xdist_group('h5py')
 @pytest.mark.parametrize( 'dtype', ['float', 'complex'] )
 def test_reconstruct_multipatch(dtype):
     bounds1   = (0.5, 1.)
@@ -602,6 +610,7 @@ def test_reconstruct_multipatch(dtype):
                         assert value1 == value2
 
 
+@pytest.mark.xdist_group('h5py')
 def test_incorrect_arg_export_to_vtk():
     domain = Square()
     space = ScalarFunctionSpace('V', domain)
@@ -646,6 +655,7 @@ def test_incorrect_arg_export_to_vtk():
     os.remove("test_incorrect_arg_export_to_vtk.h5")
 
 
+@pytest.mark.xdist_group('h5py')
 @pytest.mark.mpi
 @pytest.mark.parametrize('geometry', ['identity_2d.h5',
                                       'identity_3d.h5',
@@ -756,6 +766,7 @@ def test_parallel_export_discrete_domain(geometry, kind, space, dtype):
         os.remove("test_parallel_export_discrete_domain.h5")
 
 
+@pytest.mark.xdist_group('h5py')
 @pytest.mark.mpi
 @pytest.mark.parametrize('domain', [
     Square(),

psydac/cmd/psydac_compile.py

@@ -7,7 +7,11 @@
 The purpose of this module is to pyccelize all PSYDAC kernels, in the case
 that these were modified after an editable installation of PSYDAC.
 """
-from psydac.cmd.argparse_helpers import add_help_flag, add_version_flag
+from psydac.cmd.argparse_helpers import (
+    add_help_flag,
+    add_version_flag,
+    exit_with_error_message,
+)
 
 __all__ = (
     'setup_psydac_compile_parser',
@@ -72,4 +76,7 @@ def psydac_compile(*, language):
 
     print('Executing command:')
     print(f' {" ".join(cmd)}\n')
-    subprocess.run(cmd, shell=False)
+    result = subprocess.run(cmd, shell=False)
+
+    if result.returncode != 0:
+        exit_with_error_message('failed to compile PSYDAC kernels.')

psydac/cmd/psydac_test.py

@@ -7,7 +7,11 @@
 The purpose of this module is to pyccelize all PSYDAC kernels, in the case
 that these were modified after an editable installation of PSYDAC.
 """
-from psydac.cmd.argparse_helpers import add_help_flag, add_version_flag, exit_with_error_message
+from psydac.cmd.argparse_helpers import (
+    add_help_flag,
+    add_version_flag,
+    exit_with_error_message,
+)
 
 __all__ = (
     'setup_psydac_test_parser',
@@ -90,17 +94,17 @@ def psydac_test(*, mod, mpi, petsc, verbose, exitfirst):
         print(f'Removing existing Pytest cache directory: {cache_dir}\n', flush=True)
         shutil.rmtree(cache_dir)
 
-    # If no pytest.ini file exists in the current working directory, copy it
+    # If no pytest.toml file exists in the current working directory, copy it
     # from the parent directory of this script (which is installed with PSYDAC)
-    if not os.path.isfile('pytest.ini'):
+    if not os.path.isfile('pytest.toml'):
         script_dir = os.path.dirname(os.path.abspath(__file__))
         parent_dir = os.path.dirname(script_dir)
-        pytest_ini = os.path.join(parent_dir, 'pytest.ini')
-        if not os.path.isfile(pytest_ini):
-            exit_with_error_message(f'could not find pytest.ini file in {parent_dir}')
+        pytest_cfg = os.path.join(parent_dir, 'pytest.toml')
+        if not os.path.isfile(pytest_cfg):
+            exit_with_error_message(f'could not find pytest.toml file in {parent_dir}')
         else:
-            print(f'Copying pytest.ini from: {parent_dir}\n', flush=True)
-            shutil.copy(pytest_ini, os.getcwd())
+            print(f'Copying pytest.toml from: {parent_dir}\n', flush=True)
+            shutil.copy(pytest_cfg, os.getcwd())
 
     # Build the list of flags for pytest
     flags = []
@@ -166,4 +170,9 @@ def psydac_test(*, mod, mpi, petsc, verbose, exitfirst):
     time.sleep(0.1)  # ensure the print is shown before subprocess output
 
     # Execute the command
-    subprocess.run(cmd, shell=False, env=os.environ)
+    result = subprocess.run(cmd, shell=False, env=os.environ)
+
+    if result.returncode != 0:
+        msg = 'the PSYDAC test suite failed. '\
+              'Please check the output above for details.'
+        exit_with_error_message(msg)

psydac/cmd/tests/__init__.py

@@ -0,0 +1,5 @@
+#---------------------------------------------------------------------------#
+# This file is part of PSYDAC which is released under MIT License. See the  #
+# LICENSE file or go to https://github.com/pyccel/psydac/blob/devel/LICENSE #
+# for full license details.                                                 #
+#---------------------------------------------------------------------------#