diff --git a/.circleci/config.yml b/.circleci/config.yml index c316a305..91c5cead 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,11 @@ version: 2.1 orbs: codecov: codecov/codecov@3.2.4 +parameters: + run_full_drbuddi: + type: boolean + default: false + .dockersetup: &dockersetup docker: - image: pennlinc/qsiprep_build:26.1.7 @@ -14,8 +19,13 @@ runinstall: &runinstall if [[ -n "$CIRCLE_TAG" ]]; then VERSION="$CIRCLE_TAG" fi - git checkout $CIRCLE_BRANCH - pip install .[tests] --progress-bar off + WHEEL=$(ls dist/qsiprep-*.whl 2>/dev/null | head -n 1) + if [[ -n "$WHEEL" ]]; then + pip install "${WHEEL}[tests]" --progress-bar off + else + git checkout $CIRCLE_BRANCH + pip install .[tests] --progress-bar off + fi # Precaching fonts, set 'Agg' as default backend for matplotlib python -c "from matplotlib import font_manager" @@ -34,66 +44,60 @@ runinstall: &runinstall jobs: build: resource_class: small - docker: - - image: cimg/python:3.10.9 - working_directory: /tmp/src/qsiprep - steps: - - checkout - - run: *runinstall - - download_drbuddi_rpe_series: docker: - image: cimg/python:3.10.9 working_directory: /tmp/src/qsiprep steps: - checkout - restore_cache: - key: drbuddi_rpe_series-01 - - run: *runinstall + key: pip-{{ checksum "pyproject.toml" }} - run: - name: Download drbuddi_rpe_series test data + name: Build wheel command: | - cd /tmp/src/qsiprep/.circleci - python get_data.py $PWD/data drbuddi_rpe_series + python -m pip install --upgrade pip build --progress-bar off + python -m build --wheel - save_cache: - key: drbuddi_rpe_series-01 + key: pip-{{ checksum "pyproject.toml" }} paths: - - /tmp/src/qsiprep/.circleci/data/drbuddi_rpe_series + - /root/.cache/pip + - persist_to_workspace: + root: /tmp/src/qsiprep + paths: + - dist - download_drbuddi_epi: + download_drbuddi_data: resource_class: small docker: - - image: cimg/python:3.10.9 + - image: python:3.10-slim working_directory: /tmp/src/qsiprep steps: - checkout - restore_cache: - key: drbuddi_epi-01 - - run: *runinstall + key: drbuddi-data-01 - run: - name: Download drbuddi_epi test data + name: Download DRBUDDI test data command: | - cd /tmp/src/qsiprep/.circleci - python get_data.py $PWD/data drbuddi_epi + cd .circleci + python get_data.py $PWD/data drbuddi_rpe_series drbuddi_epi - save_cache: - key: drbuddi_epi-01 + key: drbuddi-data-01 paths: + - /tmp/src/qsiprep/.circleci/data/drbuddi_rpe_series - /tmp/src/qsiprep/.circleci/data/drbuddi_epi download_DSDTI: resource_class: small docker: - - image: cimg/python:3.10.9 + - image: python:3.10-slim working_directory: /tmp/src/qsiprep steps: - checkout - restore_cache: key: DSDTI-01 - - run: *runinstall - run: name: Download DSDTI test data command: | - cd /tmp/src/qsiprep/.circleci + cd .circleci python get_data.py $PWD/data DSDTI - save_cache: key: DSDTI-01 @@ -103,17 +107,16 @@ jobs: download_DSCSDSI: resource_class: small docker: - - image: cimg/python:3.10.9 + - image: python:3.10-slim working_directory: /tmp/src/qsiprep steps: - checkout - restore_cache: key: DSCSDSI-01 - - run: *runinstall - run: name: Download DSCSDSI test data command: | - cd /tmp/src/qsiprep/.circleci + cd .circleci python get_data.py $PWD/data DSCSDSI - save_cache: key: DSCSDSI-01 @@ -123,17 +126,16 @@ jobs: download_twoses: resource_class: small docker: - - image: cimg/python:3.10.9 + - image: python:3.10-slim working_directory: /tmp/src/qsiprep steps: - checkout - restore_cache: key: twoses-02 - - run: *runinstall - run: name: Download twoses test data command: | - cd /tmp/src/qsiprep/.circleci + cd .circleci python get_data.py $PWD/data twoses - save_cache: key: twoses-02 @@ -143,17 +145,16 @@ jobs: download_forrest_gump: resource_class: small docker: - - image: cimg/python:3.10.9 + - image: python:3.10-slim working_directory: /tmp/src/qsiprep steps: - checkout - restore_cache: key: forrestgump-01 - - run: *runinstall - run: name: Download forrest_gump test data command: | - cd /tmp/src/qsiprep/.circleci + cd .circleci python get_data.py $PWD/data forrest_gump - save_cache: key: forrestgump-01 @@ -167,6 +168,8 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: key: DSCSDSI-01 - run: *runinstall @@ -194,6 +197,8 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: key: DSDTI-01 - run: *runinstall @@ -221,6 +226,8 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: key: DSDTI-01 - run: *runinstall @@ -248,9 +255,20 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep + - restore_cache: + key: drbuddi-data-01 - restore_cache: key: DSDTI-01 - run: *runinstall + - run: + name: Skip unless run_full_drbuddi + command: | + if [ "<< pipeline.parameters.run_full_drbuddi >>" != "true" ]; then + echo "Skipping optional DRBUDDI TENSORLine job" + circleci-agent step halt + fi - run: name: Run TENSORLine with epi fmaps and DRBUDDI no_output_timeout: 2h @@ -275,9 +293,18 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: - key: drbuddi_epi-01 + key: drbuddi-data-01 - run: *runinstall + - run: + name: Skip unless run_full_drbuddi + command: | + if [ "<< pipeline.parameters.run_full_drbuddi >>" != "true" ]; then + echo "Skipping optional DRBUDDI SHORELine job" + circleci-agent step halt + fi - run: name: Run SHORELine with epi fmaps and DRBUDDI no_output_timeout: 2h @@ -302,8 +329,10 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: - key: drbuddi_rpe_series-01 + key: drbuddi-data-01 - run: *runinstall - run: name: Run Eddy with rpe series fmaps and DRBUDDI @@ -329,6 +358,8 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - run: *runinstall - run: name: Run QSIPrep on multi-shell dataset with GRE field maps @@ -354,6 +385,8 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: key: forrestgump-01 - run: *runinstall @@ -381,6 +414,8 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: key: forrestgump-01 - run: *runinstall @@ -408,6 +443,8 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: key: forrestgump-01 - run: *runinstall @@ -442,6 +479,8 @@ jobs: <<: *dockersetup steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: key: forrestgump-01 - run: *runinstall @@ -464,6 +503,8 @@ jobs: CIRCLE_CPUS: 4 steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: key: twoses-02 - run: *runinstall @@ -481,6 +522,8 @@ jobs: CIRCLE_CPUS: 4 steps: - checkout + - attach_workspace: + at: /tmp/src/qsiprep - restore_cache: key: DSDTI-01 - run: *runinstall @@ -598,14 +641,7 @@ workflows: tags: only: /.*/ - - download_drbuddi_rpe_series: - requires: - - build - filters: - tags: - only: /.*/ - - - download_drbuddi_epi: + - download_drbuddi_data: requires: - build filters: @@ -649,21 +685,25 @@ workflows: - DRBUDDI_SHORELine_EPI: requires: - - download_drbuddi_epi + - download_drbuddi_data + - build filters: tags: only: /.*/ - DRBUDDI_eddy_rpe_series: requires: - - download_drbuddi_rpe_series + - download_drbuddi_data + - build filters: tags: only: /.*/ - DRBUDDI_TENSORLine_EPI: requires: + - download_drbuddi_data - download_DSDTI + - build filters: tags: only: /.*/ @@ -736,9 +776,7 @@ workflows: - DSCSDSI - DSDTI_nofmap - DSDTI_synfmap - - DRBUDDI_SHORELine_EPI - DRBUDDI_eddy_rpe_series - - DRBUDDI_TENSORLine_EPI - maternal_brain_project - forrest_gump - forrest_gump_patch2self @@ -759,9 +797,7 @@ workflows: - DSCSDSI - DSDTI_nofmap - DSDTI_synfmap - - DRBUDDI_SHORELine_EPI - DRBUDDI_eddy_rpe_series - - DRBUDDI_TENSORLine_EPI - maternal_brain_project - forrest_gump - forrest_gump_patch2self diff --git a/.circleci/get_data.py b/.circleci/get_data.py index a85d131f..00049a5b 100755 --- a/.circleci/get_data.py +++ b/.circleci/get_data.py @@ -1,11 +1,89 @@ #!/usr/bin/env python3 -"""Download test data.""" +"""Download test data without requiring a full QSIPrep install.""" -import sys +from __future__ import annotations + +import argparse +import gzip +import lzma +import tarfile +from io import BytesIO +from pathlib import Path +from urllib.request import urlopen + +URLS = { + 'HBCD': 'https://upenn.box.com/shared/static/gn1ec8x7mtk1f07l97d0th9idn4qv3yx.xz', + 'DSCSDSI': 'https://upenn.box.com/shared/static/eq6nvnyazi2zlt63uowqd0zhnlh6z4yv.xz', + 'DSCSDSI_BUDS': 'https://upenn.box.com/shared/static/bvhs3sw2swdkdyekpjhnrhvz89x3k87t.xz', + 'DSDTI': 'https://upenn.box.com/shared/static/iefjtvfez0c2oug0g1a9ulozqe5il5xy.xz', + 'twoses': 'https://upenn.box.com/shared/static/c949fjjhhen3ihgnzhkdw5jympm327pp.xz', + 'multishell_output': ( + 'https://upenn.box.com/shared/static/hr7xnxicbx9iqndv1yl35bhtd61fpalp.xz' + ), + 'singleshell_output': ( + 'https://upenn.box.com/shared/static/9jhf0eo3ml6ojrlxlz6lej09ny12efgg.gz' + ), + 'drbuddi_rpe_series': ( + 'https://upenn.box.com/shared/static/j5mxts5wu0em1toafmrlzdndves1jnfv.xz' + ), + 'drbuddi_epi': 'https://upenn.box.com/shared/static/plyuee1nbj9v8eck03s38ojji8tkspwr.xz', + 'DSDTI_fmap': 'https://upenn.box.com/shared/static/rxr6qbi6ezku9gw3esfpnvqlcxaw7n5n.gz', + 'DSCSDSI_fmap': 'https://upenn.box.com/shared/static/l561psez1ojzi4p3a12eidaw9vbizwdc.gz', + 'maternal_brain_project': ( + 'https://upenn.box.com/shared/static/tkahg1ctipmfihvpa1gmibvcv0gb721h.xz' + ), + 'forrest_gump': 'https://upenn.box.com/shared/static/qat58an322bzzyixrrsk7cmf52q3bepq.xz', +} + + +def download_dataset(name: str, data_dir: Path) -> Path: + if name not in URLS: + raise ValueError(f'Unknown dataset {name!r}. Valid options: {", ".join(URLS)}') + + out_dir = data_dir / name + out_dir.mkdir(parents=True, exist_ok=True) + url = URLS[name] + + # Skip download if already present + if any(out_dir.iterdir()): + return out_dir + + if not url.startswith(('http://', 'https://')): + raise ValueError(f'Unexpected URL scheme for {name}: {url}') + + with urlopen(url, timeout=120) as resp: # noqa: S310 + payload = resp.read() + + if url.endswith('.xz'): + with lzma.open(BytesIO(payload)) as fobj: + with tarfile.open(fileobj=fobj) as tar: + tar.extractall(out_dir) # noqa: S202 + elif url.endswith('.gz'): + with tarfile.open(fileobj=gzip.GzipFile(fileobj=BytesIO(payload))) as tar: + tar.extractall(out_dir) # noqa: S202 + else: + raise ValueError(f'Unknown file type for {name} ({url})') + + return out_dir + + +def main(): + parser = argparse.ArgumentParser(description='Download QSIPrep test datasets') + parser.add_argument('data_dir', type=Path, help='Root directory to store datasets') + parser.add_argument( + 'datasets', + nargs='+', + help='Datasets to fetch (use "*" to download all known datasets)', + ) + args = parser.parse_args() + + datasets = list(URLS) if args.datasets == ['*'] else args.datasets + data_dir = args.data_dir + data_dir.mkdir(parents=True, exist_ok=True) + + for dataset in datasets: + download_dataset(dataset, data_dir=data_dir) -from qsiprep.tests.utils import download_test_data if __name__ == '__main__': - data_dir = sys.argv[1] - dset = sys.argv[2] - download_test_data(dset, data_dir) + main()