diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000..4817c83a --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,72 @@ +name: Docs + +on: + push: + branches: [main] + pull_request: + branches: [main] + paths: + - "docs/**" + - "src/**" + - "pyproject.toml" + - ".github/workflows/docs.yaml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v2 + with: + enable-cache: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install docs dependencies + run: uv sync --group docs + + - name: Build HTML docs + run: uv run sphinx-build -W --keep-going -b html docs/source docs/build/html + + - name: Upload Pages artifact + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + uses: actions/upload-pages-artifact@v3 + with: + path: docs/build/html + + - name: Upload preview artifact + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: html-docs + path: docs/build/html/ + retention-days: 7 + + deploy: + needs: build + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 50b2ff7d..79421287 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,8 @@ nosetests.xml # Sphinx documentation docs/_build/ +docs/build/ +docs/source/_autosummary/ verification.ipynb diff --git a/README.md b/README.md index a3849086..b322343b 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,9 @@ Run evaluation pipelines for data-driven weather models built with [Anemoi](http 2. [Credentials setup](#credentials-setup) 3. [Workspace setup](#workspace-setup) +For the rendered developer documentation, see the Sphinx site under `docs/`. +Build it locally with `uv sync --group docs && sphinx-build -b html docs/source docs/build/html`. + ## Features: - [Experiments](#experiment): compare model performance via standard and diagnostic verification - [Showcasing](#showcase): produce visual material for specific events diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..61b91dc9 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation + +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +.PHONY: help Makefile clean livehtml + +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: + rm -rf "$(BUILDDIR)" "$(SOURCEDIR)/_autosummary" + +livehtml: + sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..81c2bb21 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,29 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 00000000..c2629bf3 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,17 @@ +/* Widen the content column so wide rule tables and code samples don't wrap. */ +.wy-nav-content { + max-width: 1100px; +} + +/* Slightly tighten table cell padding for the rule reference tables. */ +.rst-content table.docutils td, +.rst-content table.docutils th { + padding: 6px 10px; +} + +/* Highlight inline literal code (filenames, run_ids) more clearly. */ +.rst-content code.literal { + background: #f6f8fa; + border: 1px solid #e1e4e8; + color: #24292e; +} diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..3b8103f4 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,97 @@ +"""Sphinx configuration for the EvalML documentation.""" + +from __future__ import annotations + +import sys +from importlib import metadata +from pathlib import Path + +# Project layout: docs/source/conf.py -> repo root is two parents up. +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "src")) + +# -- Project information ----------------------------------------------------- + +project = "EvalML" +author = "MeteoSwiss" +copyright = f"%Y, {author}" + +try: + release = metadata.version("evalml") +except metadata.PackageNotFoundError: + release = "0.0.0" +version = ".".join(release.split(".")[:2]) + +# -- General configuration --------------------------------------------------- + +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "sphinx_copybutton", + "sphinx_click", +] + +source_suffix = { + ".md": "markdown", + ".rst": "restructuredtext", +} + +myst_enable_extensions = [ + "colon_fence", + "deflist", + "linkify", + "substitution", + "tasklist", +] +myst_heading_anchors = 3 + +autosummary_generate = False +autodoc_default_options = { + "members": True, + "undoc-members": True, + "show-inheritance": True, +} +autodoc_typehints = "description" +autodoc_member_order = "bysource" + +# Napoleon picks up both Google- and NumPy-style docstrings present in src/. +napoleon_google_docstring = True +napoleon_numpy_docstring = True + +# Imports that may not be available on the docs builder (heavy scientific stack). +# We don't currently mock anything because RTD installs the project, but if a +# build env can't pull cartopy/earthkit, add modules here. +autodoc_mock_imports: list[str] = [] + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "xarray": ("https://docs.xarray.dev/en/stable/", None), + "pydantic": ("https://docs.pydantic.dev/latest/", None), + "click": ("https://click.palletsprojects.com/en/stable/", None), + "snakemake": ("https://snakemake.readthedocs.io/en/stable/", None), +} + +templates_path = ["_templates"] +exclude_patterns: list[str] = [] + +# Keep the first build green; flip to True once docstring coverage improves. +nitpicky = False + +# -- HTML output ------------------------------------------------------------- + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] +html_css_files = ["custom.css"] +html_title = f"EvalML {version}" + +html_theme_options = { + "navigation_depth": 4, + "collapse_navigation": False, + "sticky_navigation": True, + "titles_only": False, +} diff --git a/docs/source/contributing/ci.md b/docs/source/contributing/ci.md new file mode 100644 index 00000000..0493be88 --- /dev/null +++ b/docs/source/contributing/ci.md @@ -0,0 +1,95 @@ +# CI + +EvalML's GitHub Actions live under `.github/workflows/`. + +## `ci.yaml` + +The main CI workflow runs on every push to `main` and every PR targeting +`main`. + +### Job: `test` + +Matrix: + +- `os`: `ubuntu-latest` +- `python-version`: `3.11`, `3.12`, `3.13` +- `anemoi-dev`: `false` today; the leg that's `true` would install + `anemoi-inference` and `anemoi-datasets` from `main`. Keep an eye on + this if you change anything inference-adjacent. + +Steps: + +1. `astral-sh/setup-uv@v2` (with caching). +2. `actions/setup-python@v5` to install the matrix Python. +3. `uv sync --all-extras --dev` to install the project + dev tools + + the `kerchunk` extra. +4. `uv run pytest tests/`. + +### Job: `lint` + +Runs `pre-commit run --all-files --verbose`, which exercises: + +- `trailing-whitespace`, `end-of-file-fixer`. +- `ruff` (auto-fix) and `ruff-format`. +- `snakefmt workflow/`. +- The local `pydantic-schema` hook that regenerates + `workflow/tools/config.schema.json` and asserts no diff. + +## `docs.yaml` + +Triggered on push to `main`, on PR open/reopen/sync/close, and via +`workflow_dispatch`. It has three jobs: + +### `build` + +1. Installs `uv` and Python 3.12. +2. Runs `uv sync --group docs`. +3. Builds the docs with + `sphinx-build -W --keep-going -b html docs/source docs/build/html`. +4. Uploads the result as a workflow artifact (`html-docs`, 7-day + retention) so reviewers can download the rendered HTML and so the + deploy jobs can pick it up. + +`-W` turns warnings into errors, so any broken cross-reference, +malformed directive, or missing module fails the build. If the build +fails on something genuinely unfixable on the docs side (e.g. a heavy +runtime dependency that can't be installed in CI), add the offending +import to `autodoc_mock_imports` in `docs/source/conf.py`. + +The build is skipped on PR close — only the preview-cleanup needs to +run at that point. + +### `deploy-main` + +Only runs on push to `main`. Downloads the `html-docs` artifact and +publishes it to the root of the `gh-pages` branch via +[`JamesIves/github-pages-deploy-action`](https://github.com/JamesIves/github-pages-deploy-action). +The `clean-exclude: pr-preview/` setting preserves PR preview +directories so a main-branch deploy doesn't wipe open previews. + +### `preview` + +Runs on every PR event. Uses +[`rossjrw/pr-preview-action`](https://github.com/rossjrw/pr-preview-action) +to deploy the built site to +`gh-pages/pr-preview/pr-/`, and to remove that directory when +the PR is closed. The action also posts and updates a comment on the +PR with the preview URL. + +## GitHub Pages + +The rendered docs are hosted on GitHub Pages from the `gh-pages` branch. +To enable Pages on a fresh fork or new repository: + +1. Go to **Settings → Pages**. +2. Under **Build and deployment** → **Source**, select + **Deploy from a branch**. +3. Set **Branch** to `gh-pages` and **Folder** to `/ (root)`. +4. Save. + +The first push to `main` after that publishes the site at +`https://.github.io//`. PR previews appear at +`https://.github.io//pr-preview/pr-/`. + +Note: the `gh-pages` branch is created automatically by the deploy +action on the first run; you don't need to create it manually. diff --git a/docs/source/contributing/dev_setup.md b/docs/source/contributing/dev_setup.md new file mode 100644 index 00000000..b35ae362 --- /dev/null +++ b/docs/source/contributing/dev_setup.md @@ -0,0 +1,74 @@ +# Development setup + +EvalML's development workflow centres on `uv` and `pre-commit`. Get them +working once and the rest of the toolchain (Sphinx, `snakefmt`, +`ruff`, schema validation) plugs in via `pre-commit`. + +## Prerequisites + +- Python 3.11+ (managed by `uv`). +- A clone of the repo: + + ```bash + git clone https://github.com/MeteoSwiss/evalml.git + cd evalml + ``` + +## Install dev dependencies + +```bash +uv sync --dev +source .venv/bin/activate +``` + +`uv sync --dev` adds the project's runtime dependencies *and* the +`dev` group from `pyproject.toml` (currently `pre-commit`, +`snakefmt`). + +For docs work, also install the `docs` group: + +```bash +uv sync --dev --group docs +``` + +## Install pre-commit hooks + +```bash +pre-commit install +``` + +Now every `git commit` runs: + +- Trailing-whitespace and end-of-file fixes. +- `ruff` (lint + auto-fix) and `ruff-format`. +- `snakefmt workflow/`. +- A regenerate-and-diff check on `workflow/tools/config.schema.json` — + if your changes to `src/evalml/config.py` would alter the schema and + you didn't regenerate it, the hook fails. + +To run the full hook set on demand: + +```bash +pre-commit run --all-files +``` + +## Building the docs locally + +```bash +uv sync --group docs +sphinx-build -W --keep-going -b html docs/source docs/build/html +open docs/build/html/index.html +``` + +For live preview during authoring: + +```bash +sphinx-autobuild docs/source docs/build/html +``` + +The published site is hosted on GitHub Pages and rebuilt automatically +on every push to `main` by `.github/workflows/docs.yaml`. Open PRs get +a live preview at +`https://.github.io//pr-preview/pr-/`; the URL is +posted as a comment on the PR by `rossjrw/pr-preview-action` and +updates on every push. diff --git a/docs/source/contributing/style.md b/docs/source/contributing/style.md new file mode 100644 index 00000000..4b126199 --- /dev/null +++ b/docs/source/contributing/style.md @@ -0,0 +1,68 @@ +# Style and conventions + +This page collects the quick-reference conventions for code review. The +expanded version of the workflow conventions is in +[Workflow → Conventions](../workflow/conventions.md); this page focuses +on the Python source and the surrounding tooling. + +## Python + +- **Linting / formatting**: `ruff` and `ruff-format` (see + `.pre-commit-config.yaml`). Line length follows the ruff default (88). + Prefer fixing lint warnings rather than disabling them. +- **Type hints**: encouraged but not enforced. New public functions + should have type hints; sphinx-autodoc renders them in the API + reference via `autodoc_typehints = "description"`. +- **Docstrings**: Google or NumPy style; `napoleon` is enabled, so both + render correctly. The first line should be a one-sentence summary — + it ends up in `autosummary` tables. +- **Pydantic models**: live in `src/evalml/config.py`. New fields need a + `Field(..., description="...")`; the description is what surfaces in + the JSON Schema and in editor tooltips. +- **No top-level Snakemake-specific imports** in `src/`. The packages + must remain importable outside a Snakemake run so the unit tests can + exercise them. + +## Snakemake + +- Run `snakefmt workflow/` after edits. The `pre-commit` hook will catch + it but local feedback is faster. +- Rule names follow `{module}_{operation}[_{sub_operation}]`. See + [Workflow → Conventions](../workflow/conventions.md). +- Every rule declares a `log:` path under + `OUT_ROOT/logs/{rule_name}/{wildcards}.log`. +- Outputs that are directories should be paired with a sentinel `.ok` + file when downstream rules need a stable trigger. + +## Pydantic schema + +The JSON Schema at `workflow/tools/config.schema.json` is generated from +the Pydantic models. The pre-commit hook regenerates it; CI fails if the +committed schema has drifted. + +To regenerate manually: + +```bash +python src/evalml/config.py workflow/tools/config.schema.json +``` + +## Documentation + +- New pages go under `docs/source/
/.md` and need a toctree + entry in `docs/source/index.md`. +- Cross-link with markdown links (`../user_guide/cli.md`) — Sphinx + resolves them at build time and the `-W` flag in CI fails the build + if a link is broken. +- Avoid embedding line numbers in `{literalinclude}` directives unless + you really need them; prefer `:start-after:` / `:end-before:` markers + if you need a slice. +- Don't add emojis to docs unless explicitly asked. + +## Commits and PRs + +- Commit hooks must be green before a push (`pre-commit install` + ensures this). +- Keep PRs focused. A docs-only change that touches workflow rules + should be split. +- Write commit messages in the imperative present tense ("Add + ICON-CH1 patch metadata", not "Added…"). diff --git a/docs/source/contributing/testing.md b/docs/source/contributing/testing.md new file mode 100644 index 00000000..3612941e --- /dev/null +++ b/docs/source/contributing/testing.md @@ -0,0 +1,77 @@ +# Testing + +EvalML uses `pytest`, with tests split into `tests/unit/` and +`tests/integration/`. The CI matrix covers Python 3.11–3.13 and a +"dev anemoi" leg that pulls `anemoi-inference` and `anemoi-datasets` +straight from `main`. + +## Running tests + +```bash +# Everything (the default). +uv run pytest tests/ + +# Unit tests only. +uv run pytest tests/unit + +# Integration tests only. +uv run pytest tests/integration + +# Skip slow tests. +uv run pytest -m "not longtest" +``` + +The `longtest` marker is configured in `pyproject.toml` for tests that +touch external services or take more than a few seconds. + +## What's covered today + +`tests/unit/` ships: + +| File | Under test | +| --- | --- | +| `test_config.py` | Pydantic model loading of every example config; baseline ID derivation. | +| `test_colormaps.py` | NCL colormap parser, boundary norms, default colormap smoke tests. | +| `test_run_identity.py` | `env_id` / `run_id` hashing logic in `common.smk` (via the constants exported from `evalml.config`). | +| `test_spatial_mapping.py` | Spherical NN mapping, grid indexing, forecast-to-truth coordinate alignment. | + +`tests/integration/test_experiment.py` is a placeholder for end-to-end +experiment validation — flesh it out as workflow logic stabilises. + +## Fixtures + +`tests/conftest.py` loads two example configs as fixtures so tests don't +need to repeatedly parse YAML: + +- `forecaster_config_dict` — `config/forecasters-co2.yaml` +- `interpolator_config_dict` — `config/interpolators-co2.yaml` + +Use these whenever a test needs a realistic config. They are loaded as +plain `dict`s so individual tests can mutate them before passing them to +`ConfigModel.model_validate(...)`. + +## Adding a test + +- Put it under `tests/unit/` if it can run in under a second without + network or filesystem state outside the repo. Otherwise put it under + `tests/integration/` and consider marking it `@pytest.mark.longtest`. +- Reuse the fixtures in `conftest.py` rather than re-loading example + configs. +- For Snakemake-touching tests, prefer testing the underlying Python + function rather than running Snakemake itself — most of the + hash/registry logic in `common.smk` is small enough to test in + isolation. +- For new metrics or loaders, add at least one test that exercises the + end-to-end shape of the returned `xarray.Dataset`. + +## What CI runs + +[`.github/workflows/ci.yaml`](../../../.github/workflows/ci.yaml) has two +jobs: + +- **`test`** — matrix over Python 3.11/3.12/3.13. The "anemoi-dev" leg is + defined in the matrix but currently set to `false` only; switching it + on in the matrix turns on a job that installs anemoi from `main`. +- **`lint`** — runs `pre-commit run --all-files --verbose`. + +The docs build is its own workflow (see [CI](ci.md)). diff --git a/docs/source/getting_started/auth.md b/docs/source/getting_started/auth.md new file mode 100644 index 00000000..a15a5af0 --- /dev/null +++ b/docs/source/getting_started/auth.md @@ -0,0 +1,41 @@ +# Credentials setup + +Some experiments are stored on the ECMWF-hosted MLflow server at +[https://mlflow.ecmwf.int](https://mlflow.ecmwf.int). To access these runs in +the evaluation workflow, you need to authenticate using a valid token. + +## One-time login + +Run the following commands **once** to obtain a token: + +```bash +uv pip install anemoi-training --no-deps +anemoi-training mlflow login --url https://mlflow.ecmwf.int +``` + +You will be prompted to paste a seed token obtained from +. + +## Token lifetime + +After the first login, the token is stored locally and reused for subsequent +runs. Tokens are valid for **30 days**, but every successful training or +evaluation run within that window automatically extends the token by another +30 days. + +It's good practice to run the login command before executing the workflow to +ensure the token is still valid — a stale token will surface as an MLflow +authentication error during `inference_get_checkpoint`. + +## Other checkpoint sources + +EvalML accepts checkpoints from three sources, classified by URL in +[common.smk](../workflow/inference.md): + +- **MLflow** (`mlflow.ecmwf.int`, `service.meteoswiss.ch`, + `servicedepl.meteoswiss.ch`). +- **Hugging Face** (`huggingface.co//blob//.ckpt`) — uses + `uvx hf download` and follows the standard Hugging Face authentication + (set `HF_TOKEN` if the repo is gated). +- **Local paths** — a filesystem path that exists is treated as a local + checkpoint. diff --git a/docs/source/getting_started/installation.md b/docs/source/getting_started/installation.md new file mode 100644 index 00000000..9ac67e2a --- /dev/null +++ b/docs/source/getting_started/installation.md @@ -0,0 +1,55 @@ +# Installation + +EvalML is distributed as a `uv`-managed Python project. + +## Prerequisites + +- A Linux environment; the workflow has been tested on CSCS Balfrin. +- [`uv`](https://github.com/astral-sh/uv) — install with: + ```bash + curl -LsSf https://astral.sh/uv/install.sh | sh + ``` +- A working `git`, `mksquashfs`, and `squashfs-mount` if you intend to run the + inference rules on a SLURM cluster. + +## Installing the project + +Clone the repository and create the virtual environment with `uv`: + +```bash +git clone https://github.com/MeteoSwiss/evalml.git +cd evalml +uv sync +source .venv/bin/activate +``` + +`uv sync` installs the runtime dependencies declared in `pyproject.toml`. Add +`--dev` to also install pre-commit and `snakefmt`, and `--group docs` to install +the documentation toolchain (Sphinx, theme, MyST). Both groups stack: + +```bash +uv sync --dev --group docs +``` + +## Verifying the install + +```bash +evalml --help +``` + +You should see the four top-level subcommands (`experiment`, `showcase`, +`sandbox`, `make`). If the entry point is missing, the project is not +installed in the active environment — re-run `uv sync` and ensure +`source .venv/bin/activate` was successful. + +## Optional: Kerchunk extras + +If you plan to read sharded Zarr datasets via Kerchunk references, install the +`kerchunk` optional group: + +```bash +uv sync --extra kerchunk +``` + +This pulls `kerchunk`, `zarr<3.0.0`, `fastparquet`, and `ujson`. It is *not* +required for standard experiments. diff --git a/docs/source/getting_started/quickstart.md b/docs/source/getting_started/quickstart.md new file mode 100644 index 00000000..5c2f50e2 --- /dev/null +++ b/docs/source/getting_started/quickstart.md @@ -0,0 +1,128 @@ +# Quickstart + +This page walks through running your first experiment end-to-end. It assumes +you have already followed [Installation](installation.md) and +[Credentials setup](auth.md). + +## 1. Pick an example config + +The repository ships several end-to-end configurations under `config/`: + +| File | What it does | +| --- | --- | +| `config/forecasters-co2.yaml` | COSMO-2 forecaster comparison on storm cases | +| `config/forecasters-ich1.yaml` | ICON-CH1 single-forecaster evaluation | +| `config/interpolators-co2.yaml` | M-2 temporal downscaler on COSMO-2 data | + +A walkthrough of each is in [Example configs](../user_guide/examples.md). + +## 2. Run the experiment + +```bash +evalml experiment config/forecasters-co2.yaml --report +``` + +What `evalml` does, step by step: + +1. **Validate** the YAML against the Pydantic `ConfigModel` (see + [Configuration](../user_guide/configuration.md)). +2. **Build** a Snakemake invocation using the `profile:` block. +3. **Launch** Snakemake with the `experiment_all` target. + +`--report` additionally tells Snakemake to produce a self-contained HTML +report after a successful run. + +## 3. Useful flags + +```bash +# Don't execute, just show what would run. +evalml experiment config.yaml --dry-run + +# Render the dependency DAG to dag.svg via kroki.io. +evalml experiment config.yaml --dag + +# Render only the rule graph. +evalml experiment config.yaml --rulegraph + +# Pass arbitrary options to Snakemake after `--`. +evalml experiment config.yaml -- --jobs 1 --use-conda +``` + +The full option matrix is documented in [The evalml CLI](../user_guide/cli.md). + +## 4. Inspect the results + +```bash +ls output/results/ +``` + +Each run produces a directory named +`{YYYYMMDD}_{config_label}_{config_hash}/` containing the rendered dashboard +(`dashboard/dashboard.html`) and the verification plots (`plots/*.png`). +Forecast animations and meteograms only appear if you used +`evalml showcase` instead of `evalml experiment`. + +## 5. What lives under `output/data/` + +`output/results/` only holds the polished, shareable artefacts. Almost +everything else lands in `output/data/`, which is roughly three trees: + +```text +output/data/ +├── runs// # one per checkpoint + dep set +│ ├── inference-last.ckpt # symlink / copy of the checkpoint +│ ├── requirements.txt # auto-derived from MLflow metadata +│ ├── venv.squashfs # built once, mounted on compute nodes +│ └── / # one per inference config + steps +│ ├── summary.md # human-readable run description +│ ├── verif_aggregated.nc # aggregated metrics across init times +│ └── / # one per forecast init +│ ├── config.yaml # rendered inference config +│ ├── grib/ # raw GRIB forecast output +│ └── verif.nc # per-init metrics +├── baselines// # one per `baseline:` entry +│ ├── verif_aggregated.nc +│ └── /verif.nc +└── observations/ # cached PeakWeather etc. +``` + +A few things worth knowing as you start poking around: + +- `data/runs//` is **shared** across every run that has the same + checkpoint and the same extra dependencies. Two runs that only differ + in `steps` or in the inference config will share the (expensive) + `venv.squashfs` but each get their own `/` subdirectory. +- `verif.nc` files are plain `xarray` datasets and can be opened with + `xr.open_dataset` for ad-hoc analysis. The full schema is documented + in {ref}`Outputs and wildcards → What's inside a verif.nc `. +- `grib/` directories are large. If disk pressure matters, prune + `data/runs////grib/` for old init + times once `verif.nc` has been written — the metrics rule does not + need them again. +- The full layout (logs, sentinel `.ok` files, every wildcard) is in + [Outputs and wildcards](../user_guide/outputs.md). + +(iterate-without-re-running-everything)= +## 6. Iterate without re-running everything + +Snakemake caches outputs based on file hashes, so re-running with a tweaked +config will only redo affected rules. If you change something that EvalML +hashes into `run_id` (steps, inference config, dependencies), a new +sub-directory is created and the existing one is left untouched. See +[Outputs and wildcards](../user_guide/outputs.md) for the rules behind this. + +**Caveat:** only the `checkpoint` **path/URL** enters the hash — not its +contents. If a checkpoint is mutated in place while the path stays the +same, EvalML reuses the cached environment and outputs. To force a +rebuild: + +```bash +# Force everything to re-run, even if the outputs look up-to-date. +evalml experiment config.yaml -- -F + +# Force just one rule (and its downstream dependents) to re-run. +evalml experiment config.yaml -- -R verification_metrics +``` + +See [CLI → Forcing re-runs](../user_guide/cli.md#forcing-re-runs) for the +full list of force flags. diff --git a/docs/source/getting_started/workspace.md b/docs/source/getting_started/workspace.md new file mode 100644 index 00000000..0c83fc17 --- /dev/null +++ b/docs/source/getting_started/workspace.md @@ -0,0 +1,53 @@ +# Workspace setup + +By default, every artefact produced by the workflow is written under `output/` +in your working directory. If the working directory is in your home directory on an HPC cluster, this is probably not the optimal place because `output/` quickly grows past the home-directory quota. + +## Recommended: symlink output to scratch + +```bash +mkdir -p $SCRATCH/evalenv/output +ln -s $SCRATCH/evalenv/output output +``` + +This way the data lives on scratch but you can still browse it from your IDE +as if it were a local directory. + +## Editor support + +If you use VSCode, install the **YAML** extension. It will pick up the +JSON-Schema reference in the example configs: + +```yaml +# yaml-language-server: $schema=../workflow/tools/config.schema.json +``` + +and provide: + +- Hover documentation pulled from the `Field(description=...)` strings in + `src/evalml/config.py`. +- Autocompletion for valid keys and enum-like fields (e.g. `frequency`). +- Inline validation of types and required fields. + +The JSON Schema is regenerated by a `pre-commit` hook (see +[Contributing → Style](../contributing/style.md)) so it never drifts from the +Pydantic source of truth. + +## Where things land + +After a successful run, your `output/` (or scratch symlink) will look roughly +like: + +```text +output/ +├── data/ # intermediate per-env / per-run / per-init artefacts +│ ├── runs/ +│ ├── baselines/ +│ └── observations/ +├── logs/ # one sub-directory per rule +└── results/ # final products: dashboards, plots, animations +``` + +The full layout — including wildcard conventions, sentinel `.ok` files, and +the `{env_id}/{config_hash}` split inside `data/runs/` — is documented in +[Outputs and wildcards](../user_guide/outputs.md). diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 00000000..f75c71ec --- /dev/null +++ b/docs/source/index.md @@ -0,0 +1,100 @@ +# EvalML + +EvalML runs evaluation pipelines for data-driven weather models built with +[Anemoi](https://anemoi.readthedocs.io/). It provides a small Click-based CLI +that drives a Snakemake workflow capable of: + +- **Experiments** — compare model checkpoints against baselines and truth data + through standard and diagnostic verification. +- **Showcases** — produce visual material (forecast animations, meteograms) + for specific weather events. +- **Sandboxes** — package isolated inference environments for any Anemoi + checkpoint, suitable for development or sharing with collaborators. + +These docs are written for developers who need to understand, extend, or debug +the pipeline. If you only want to *run* an experiment, the +[Quickstart](getting_started/quickstart.md) is the fastest path; if you want to +understand how a rule works or how `run_id` is hashed, the +[Workflow](workflow/overview.md) section is for you. + +## How the pieces fit together + +```text +YAML config ─► evalml CLI ─► Snakemake ─► rules/*.smk ─► scripts/*.py ─► src/ ─► OUT_ROOT/ +``` + +The Click CLI (`src/evalml/cli.py`) validates the YAML config against the +Pydantic models in `src/evalml/config.py`, builds a Snakemake invocation, and +launches the appropriate top-level target (`experiment_all`, `showcase_all`, +`sandbox_all`). Snakemake then resolves the dependency DAG defined in +`workflow/rules/*.smk`, executing rules whose scripts import the four +src-layout packages: `evalml`, `verification`, `data_input`, `plotting`. + +```{toctree} +:caption: Getting started +:maxdepth: 2 + +getting_started/installation +getting_started/auth +getting_started/workspace +getting_started/quickstart +``` + +```{toctree} +:caption: User guide +:maxdepth: 2 + +user_guide/cli +user_guide/configuration +user_guide/outputs +user_guide/examples +``` + +```{toctree} +:caption: Workflow +:maxdepth: 2 + +workflow/overview +workflow/data +workflow/inference +workflow/verification +workflow/plotting +workflow/reporting +workflow/conventions +``` + +```{toctree} +:caption: Python API +:maxdepth: 2 + +modules/index +modules/evalml +modules/verification +modules/data_input +modules/plotting +``` + +```{toctree} +:caption: Reference +:maxdepth: 2 + +reference/config_schema +reference/resources +reference/glossary +``` + +```{toctree} +:caption: Contributing +:maxdepth: 2 + +contributing/dev_setup +contributing/testing +contributing/ci +contributing/style +``` + +## Indices + +- {ref}`genindex` +- {ref}`modindex` +- {ref}`search` diff --git a/docs/source/modules/data_input.md b/docs/source/modules/data_input.md new file mode 100644 index 00000000..43ea9615 --- /dev/null +++ b/docs/source/modules/data_input.md @@ -0,0 +1,61 @@ +# `data_input` + +`data_input` is the central place for loading forecasts and ground-truth +into the verification pipeline. Nearly every loader returns an +`xarray.Dataset` with a consistent schema (parameter names, units, time +coordinates), so downstream code can treat forecast and truth uniformly. + +## Package contents + +```{eval-rst} +.. automodule:: data_input + :members: +``` + +## Loader dispatch + +The package exposes two top-level dispatchers: + +- `load_truth_data(root, reftime, steps, params)` — picks + `load_analysis_data_from_zarr` for `.zarr` paths and + `load_obs_data_from_peakweather` for PeakWeather caches. +- `load_forecast_data(root, reftime, steps, params)` — picks + `load_fct_data_from_grib` for GRIB directories (EvalML inference + output) and `load_baseline_from_zarr` for Zarr-based baselines. + +These dispatchers are what verification scripts call; the underlying +loaders are fine to use directly when you need to short-circuit the +dispatch (e.g. inside a notebook). + +## `parse_steps` + +`parse_steps("0/120/6")` returns `[0, 6, 12, ..., 120]`. The same +`start/end/step` format is validated by `RunConfig.steps`. + +## Conventions and pitfalls + +- **Variable renaming**: `load_analysis_data_from_zarr` renames Anemoi + variables to their COSMO equivalents (`t_2m → T_2M`, etc.). Add new + mappings to the script-local rename table when you need them. +- **TOT_PREC unit conversion**: the analysis Zarr stores precipitation in + metres; `load_analysis_data_from_zarr` multiplies by 1000 to convert to + mm, the canonical unit downstream. +- **TOT_PREC disaggregation**: GRIB and Zarr loaders both expect + cumulative-from-start precipitation (the + `accumulate_from_start_of_forecast` post-processor must be enabled in + anemoi-inference) and disaggregate it with `.diff("lead_time")`. A + sanity check raises if `min(.diff())` is significantly negative — + that's the signature of data that's already per-step accumulated and + would be garbled by a second disaggregation. +- **Lead-time selection up-front**: both forecast and baseline loaders + now restrict to the requested lead times *before* disaggregation, so + sub-step baselines (e.g. hourly baseline against 6-hourly forecast) + produce the correct accumulation window. +- **Unit conversions**: `T_2M` from PeakWeather is converted to Kelvin in + `load_obs_data_from_peakweather`, matching the ML model output. Other + variables retain their source units. +- **Missing valid times**: `_select_valid_times(ds, times)` warns instead + of raising when a requested step is missing, which is intentional — + baseline archives sometimes have gaps and forcing a hard error would + block whole experiments. If you need stricter behaviour, do an + explicit check on the returned dataset. diff --git a/docs/source/modules/evalml.md b/docs/source/modules/evalml.md new file mode 100644 index 00000000..27bda5ab --- /dev/null +++ b/docs/source/modules/evalml.md @@ -0,0 +1,58 @@ +# `evalml` package + +The `evalml` package contains the user-facing CLI, the Pydantic +configuration models, and a small handful of helpers used by Snakemake +scripts. + +## Click CLI + +The CLI is auto-rendered from the source on the +[The evalml CLI](../user_guide/cli.md) page — go there for the full +command tree with narrative. + +## CLI plumbing (`evalml.cli`) + +The Click commands themselves are thin; the interesting bits are the +`workflow_options` decorator, `execute_workflow`, and the +`generate_graph` helper that talks to kroki.io. + +```{eval-rst} +.. automodule:: evalml.cli + :members: + :exclude-members: cli, experiment, showcase, sandbox, make +``` + +## Configuration models (`evalml.config`) + +Every YAML config is validated through `ConfigModel`. The hierarchy: + +- `ConfigModel` — top-level container. Holds an optional + `thresholds: dict[param, dict[op, list[float]]]` field for categorical + verification, validated to accept only the operator keys + `gt`, `ge`, `lt`, `le`, `eq`, `ne`. +- `Dates` / `ExplicitDates` — date specification. +- `RunConfig` (abstract base) → `ForecasterConfig`, `InterpolatorConfig`. +- `BaselineConfig`, `TruthConfig`, `Stratification`, `Locations`. +- `Dashboard` — settings for the report dashboard (currently the + `stratification` axes to expose: any of `season`, `region`, + `init_hour`). +- `Profile` → `GlobalResources`, `DefaultResources`. +- `InferenceResources` — optional per-run override. + +```{eval-rst} +.. automodule:: evalml.config + :members: + :show-inheritance: + :exclude-members: model_config, model_fields, model_computed_fields +``` + +## Helpers (`evalml.helpers`) + +```{eval-rst} +.. automodule:: evalml.helpers + :members: +``` + +`setup_logger` is the recommended way to configure logging inside Snakemake +scripts — call it once near the top of a script with the rule's `log[0]` +path so all script output ends up in the rule's log file. diff --git a/docs/source/modules/index.md b/docs/source/modules/index.md new file mode 100644 index 00000000..d369a534 --- /dev/null +++ b/docs/source/modules/index.md @@ -0,0 +1,34 @@ +# Python API + +EvalML's Python source uses a flat src-layout with four packages: + +| Package | Purpose | +| --- | --- | +| [`evalml`](evalml.md) | The Click CLI and Pydantic configuration models. | +| [`verification`](verification.md) | Spatial verification metrics and shapefile-based region masks. | +| [`data_input`](data_input.md) | Loaders for Zarr, GRIB, and PeakWeather data. | +| [`plotting`](plotting.md) | Earthkit/cartopy plotting helpers and colormap defaults. | + +Each subpage mixes prose with `autosummary` and `automodule` directives, so +the reference is always in sync with the source. If you need a single +flat index of every member, `genindex` is generated by Sphinx — see the +bottom of the [landing page](../index.md). + +## Conventions + +- Modules are imported by Snakemake scripts, not by users directly. The + CLI is the only externally-facing API. +- All Pydantic models live in `evalml.config`. They use + `model_config = {"extra": "forbid"}` at the top level so misspelled + fields fail validation immediately. +- `verification` and `data_input` deliberately avoid Snakemake-specific + imports so they can be unit-tested without a workflow runtime. +- `plotting` depends on `cartopy` and `earthkit-plots`; importing it from + a thin environment may pull in heavy native dependencies. + +## Cross-references + +`intersphinx` is configured for the Python standard library, NumPy, +xarray, Pydantic, Click, and Snakemake. Anything documented in those +projects can be cross-linked with a regular `:func:`, `:class:`, or +`:obj:` reference. diff --git a/docs/source/modules/plotting.md b/docs/source/modules/plotting.md new file mode 100644 index 00000000..d2cb0ec3 --- /dev/null +++ b/docs/source/modules/plotting.md @@ -0,0 +1,85 @@ +# `plotting` + +The `plotting` package wraps `earthkit-plots` and `cartopy` for fast, +consistent geographic figures. It is imported by the Marimo notebooks +under `workflow/scripts/plot_*.mo.py` and by the dashboard pipeline. + +## Top-level API + +```{eval-rst} +.. automodule:: plotting + :members: +``` + +`StatePlotter` precomputes a Delaunay triangulation in `__init__`, so +repeated calls to `plot_field` on the same grid are inexpensive. For +orthographic projections, only the visible hemisphere is triangulated — +this is handled by the cached `_orthographic_tri` property. + +`DOMAINS` is a small registry mapping domain names (`globe`, `europe`, +`centraleurope`, `switzerland`) to bounding boxes and projections, so +plotting code can stay free of magic numbers: + +```python +from plotting import DOMAINS, StatePlotter + +plotter = StatePlotter(lon, lat, out_dir) +fig = plotter.init_geoaxes(**DOMAINS["switzerland"]) +plotter.plot_field(fig.subplots[0, 0], field, title="T2m") +``` + +## Colormap loader + +```{eval-rst} +.. automodule:: plotting.colormap_loader + :members: +``` + +NCL-style `.ct` files live under `resources/report/plotting/`. The loader +returns a dict containing a `ListedColormap`, a `BoundaryNorm`, and the +list of bounds — suitable for direct use with Matplotlib's contour +functions. + +## Default colormaps per parameter + +```{eval-rst} +.. automodule:: plotting.colormap_defaults + :members: +``` + +`CMAP_DEFAULTS` is the lookup table consumed by the Marimo plotting +notebooks. Keys are upper-case parameter names (`T_2M`, `V_10M`, +`TOT_PREC_1H`, `TOT_PREC_6H`, `SP`, `FI_850`, …) and values bundle +`cmap`, `norm`, `units`, and (where appropriate) `vmin`/`vmax`. +Precipitation is keyed per accumulation window because the colour +levels need to scale with the integration period — the `plot_forecast_frame` +rule passes an `--accu` value (in hours) so the notebook can pick the +matching entry. Unknown keys fall back to a viridis colormap with a +warning, so an experiment that tries to plot a brand-new parameter will +produce something readable while you fix the mapping. + +## State helpers (`plotting.compat`) + +```{eval-rst} +.. automodule:: plotting.compat + :members: +``` + +`load_state_from_grib` and `load_state_from_raw` return a dict shaped +like: + +```python +{ + "forecast_reference_time": ..., + "valid_time": ..., + "longitudes": ..., + "latitudes": ..., + "lam_envelope": GeoSeries, + "fields": {"T_2M": np.ndarray, ...}, +} +``` + +This shape mirrors the in-memory state that anemoi-inference produces, +which is why these helpers are grouped under `compat` — they exist to +let plotting code consume the same shape regardless of whether a forecast +came back as a GRIB directory on disk or a `.npy` snapshot. diff --git a/docs/source/modules/verification.md b/docs/source/modules/verification.md new file mode 100644 index 00000000..63947ba2 --- /dev/null +++ b/docs/source/modules/verification.md @@ -0,0 +1,72 @@ +# `verification` + +The `verification` package implements forecast-vs-observation metrics and +the spatial machinery (shapefile masks, nearest-neighbour mapping) needed +to align them. It is consumed by +[verification_metrics](../workflow/verification.md) and +[verification_aggregation](../workflow/verification.md), and is also +unit-tested in `tests/unit/test_spatial_mapping.py`. + +The package is split into two modules: + +- **`verification`** (package `__init__.py`) — shapefile masks, the public + `verify` entry point, continuous and categorical score helpers, and the + `decode_metric` label translator. +- **`verification.spatial`** — spherical nearest-neighbour mapping + utilities (`spherical_nearest_neighbor_indices`, + `nearest_grid_yx_indices`, `map_forecast_to_truth`). + +## Top-level package (`verification`) + +```{eval-rst} +.. automodule:: verification + :members: + :show-inheritance: +``` + +## Spatial mapping (`verification.spatial`) + +```{eval-rst} +.. automodule:: verification.spatial + :members: + :show-inheritance: +``` + +## Notes on the API + +- `verify(...)` returns a single `xarray.Dataset` with named regions on a + `region` dimension. A reserved `all` region is always present. Pass + `regions=None` to compute over the full grid only. +- Continuous metrics (BIAS, MSE, MAE, CORR) are computed via the + [`scores`](https://scores.readthedocs.io/) library. Statistics + (mean, var, min, max) come from xarray directly. +- Pass `threshold_dict={"param": {"gt": [v1, v2], ...}, ...}` to + additionally produce per-`(parameter, operator, value)` contingency + tables. They land on a `threshold` dimension whose values are encoded + as `{op}_{value}` (e.g. `gt_0p001`). Use `decode_metric` to render + the encoded labels back to human form (`gt 0.001`). +- `map_forecast_to_truth(...)` is the right function to align a forecast + expressed on grid A onto the truth grid B before metric computation. + It uses `spherical_nearest_neighbor_indices` under the hood, so it does + not suffer from the lat/lon distortion of a plain Euclidean + nearest-neighbour search. +- `ShapefileSpatialAggregationMasks` is the only currently-shipped + concrete subclass of `SpatialAggregationMasks`. To support a different + region geometry (e.g. raster masks), subclass `AggregationMasks` and + implement `get_masks`. + +## Adding a metric + +1. **Continuous**: extend the dataset built in `_compute_scores` with a + new key (typically using a `scores.continuous.*` helper). +2. **Categorical**: extend `_binary_confusion_matrix` or add a sibling + helper that produces a (`threshold`,)-shaped DataArray, then attach + it to the result in `_compute_scores`. +3. Update `workflow/scripts/verification_aggregation.py` so the new + field survives aggregation across initialisation times. +4. Update `report_experiment_dashboard.py` and the dashboard template if + the metric should be plottable. +5. Add a test under `tests/unit/`. The existing + `tests/unit/test_spatial_mapping.py` covers the spatial helpers; new + metric tests should live in a new file (e.g. + `tests/unit/test_metrics.py`). diff --git a/docs/source/reference/config_schema.md b/docs/source/reference/config_schema.md new file mode 100644 index 00000000..0856a969 --- /dev/null +++ b/docs/source/reference/config_schema.md @@ -0,0 +1,38 @@ +# Config JSON Schema + +The repository ships a JSON Schema generated from the Pydantic models in +`src/evalml/config.py`. It lives at `workflow/tools/config.schema.json` +and is regenerated automatically by a `pre-commit` hook so it never drifts +from the Python source of truth. + +To regenerate it manually: + +```bash +python src/evalml/config.py workflow/tools/config.schema.json +``` + +The same schema is also exposed programmatically: + +```python +from evalml.config import generate_config_schema +schema = generate_config_schema() +``` + +## Using the schema in your editor + +Add a YAML language-server hint at the top of any config file: + +```yaml +# yaml-language-server: $schema=../workflow/tools/config.schema.json +``` + +VSCode (with the YAML extension), Neovim (with `coc-yaml` or +`yaml-language-server`), and most modern editors will then surface +hover-docs, autocompletion, and inline validation pulled directly from +the `Field(description=...)` strings. + +## Full schema + +```{literalinclude} ../../../workflow/tools/config.schema.json +:language: json +``` diff --git a/docs/source/reference/glossary.md b/docs/source/reference/glossary.md new file mode 100644 index 00000000..c45e36d5 --- /dev/null +++ b/docs/source/reference/glossary.md @@ -0,0 +1,80 @@ +# Glossary + +```{glossary} +candidate + A run that participates in an experiment's verification, dashboard, + and plots. Set internally by `register_run(... as_candidate=True)`. + Forecasters and temporal downscalers ('interpolator') added directly + under `runs:` are candidates by default; nested upstream forecasters registered as dependencies of a temporal downscaler are *not* candidates + unless they were also explicitly listed. + +dependency run + A run registered indirectly because another run depends on it (e.g. + the forecaster nested inside an `interpolator:` block). Its env and + output directories are built but it does not appear in the + dashboard. + +env_id + `{model_type}-{model_id}-{env_hash}` (with `-on-{forecaster_env}` + appended for temporal downscalers ('interpolator')). Identifies the inference environment (venv, squashfs). Two runs that differ only in inference config or `steps` share an `env_id`. + +run_id + `{env_id}/{config_hash}`. Identifies a specific run configuration. + Each unique `run_id` has its own output directory under + `data/runs/`. + +baseline + A reference forecast (typically COSMO-E or another operational NWP + archive) that is read from disk rather than computed by EvalML. The + `baseline_id` is `Path(root).stem`. + +truth + The ground-truth dataset used in verification. Either an analysis + Zarr or a PeakWeather observations cache. + +OUT_ROOT + The path-based shorthand for `locations.output_root` from the YAML + config. Used everywhere in the Snakefile. + +experiment_name + `{YYYYMMDD}_{config_label}_{config_hash}`. Identifies one + invocation of the workflow. Used as the `{experiment}` (and + `{showcase}`) wildcard. + +config_hash + Short SHA-256 of the merged config plus every run's env- and + run-specific hashes. Computed by `master_hash()` in + `workflow/rules/common.smk`. + +env_hash + Short SHA-256 of the fields in `RunConfig.ENV_FIELDS` + (`checkpoint`, `extra_requirements`, + `disable_local_eccodes_definitions`). Computed by + `env_entry_hash()`. + +run_hash + Short SHA-256 of `steps` plus the inference config YAML contents. + Computed by `run_specific_hash()`. + +inference sandbox + A zip produced by `inference_create_sandbox` that bundles a + checkpoint, a `requirements.txt`, an inference config, and a + rendered README. Suitable for handing a checkpoint plus a + reproducible runtime to an external collaborator. + +squashfs image + A read-only filesystem image (`venv.squashfs`) of the inference + venv. `inference_execute` mounts it on the SLURM compute node via + `squashfs-mount` and activates it as `/user-environment`. + +candidate filtering + `collect_all_candidates()` returns only runs with + `_is_candidate=True`. The `experiment_all` and `showcase_all` target + rules iterate over candidates only, so adding a dependency run does + not produce extra dashboard entries. + +stratification region + A polygon defined by a shapefile under `stratification.root`, used + to compute regional verification scores. The hardcoded `all` region + covers the entire grid and is always present. +``` diff --git a/docs/source/reference/resources.md b/docs/source/reference/resources.md new file mode 100644 index 00000000..2df1d71c --- /dev/null +++ b/docs/source/reference/resources.md @@ -0,0 +1,79 @@ +# Resources + +The `resources/` directory ships static assets used by the workflow at +runtime. Nothing under `resources/` is generated; everything is checked +into git and is part of the repository's reproducibility contract. + +## `resources/inference/` + +Layout: + +```text +resources/inference/ +├── configs/ # Anemoi inference YAML templates +├── metadata/ # ICON-CH1 patch metadata +├── templates/ # GRIB output templates +└── sandbox/ # Jinja2 README for sandbox zips +``` + +### `configs/` + +Inference templates for forecasters and temporal downscaler ('interpolator'), parameterised by model family (COSMO-2 / ICON-CH1 / IFS) and grid (regional / global). Files include: + +- `forecaster.yaml` and `interpolator.yaml` — the defaults referenced by + `ForecasterConfig` and `InterpolatorConfig` when no `config` is given. +- `sgm-forecaster-global.yaml`, `sgm-forecaster-global_trimedge.yaml` — + global forecaster variants. +- ICON-CH1- and COSMO-2-specific templates for regional inference. + +A run picks one of these via `config:` in the YAML; the workflow then +renders run-specific values (lead time, paths) on top. + +### `metadata/` + +ICON-CH1 patch metadata for operational and multi-dataset configurations. +These files carry information that the inference pipeline expects but +which is not present in every checkpoint's MLflow metadata. + +### `templates/` + +GRIB output templates used to give inference output the right edition, +table version, and indexing for COSMO-1E, COSMO-2, ICON-CH1, and IFS. An +`index.yaml` per family points at the matching template files. + +### `sandbox/` + +A Jinja2 README template (`readme.md.jinja2`) that's rendered into the +sandbox zip created by `inference_create_sandbox`. The rendered README +explains how to extract and use the sandbox. + +## `resources/report/` + +Layout: + +```text +resources/report/ +├── dashboard/ # HTML template + script.js for the dashboard +└── plotting/ # NCL .ct colormaps for plotting +``` + +### `dashboard/` + +The dashboard template (`template.html.jinja2`) and its front-end +JavaScript (`script.js`) are read by `report_experiment_dashboard.py` +and embedded into a self-contained HTML file in +`results/{experiment}/dashboard/`. The script tag is inlined; no +external CDN dependencies. + +### `plotting/` + +NCL-style `.ct` colormap files for weather fields (T2M, UV winds, RH at +various levels). Loaded by `plotting.colormap_loader.load_ncl_colormap` +and exposed through `plotting.colormap_defaults.CMAP_DEFAULTS`. + +To add a new colormap: + +1. Drop a `.ct` file under `resources/report/plotting/`. +2. Register it in `CMAP_DEFAULTS` in + `src/plotting/colormap_defaults.py` against the parameter name. +3. Verify with `pytest tests/unit/test_colormaps.py`. diff --git a/docs/source/user_guide/cli.md b/docs/source/user_guide/cli.md new file mode 100644 index 00000000..39c02f0a --- /dev/null +++ b/docs/source/user_guide/cli.md @@ -0,0 +1,96 @@ +# The `evalml` CLI + +The CLI is a thin Click wrapper around Snakemake. Its job is to validate the +config, build a Snakemake command, and pass control to Snakemake. The +implementation lives in [src/evalml/cli.py](../../../src/evalml/cli.py). + +## Auto-generated reference + +The full command tree, including options and arguments, is rendered directly +from the Click definitions: + +```{eval-rst} +.. click:: evalml.cli:cli + :prog: evalml + :nested: full +``` + +## How a command runs + +All four subcommands (`experiment`, `showcase`, `sandbox`, `make`) call into +the same `execute_workflow` helper. The high-level flow is: + +1. Parse the YAML file with `load_yaml`, then validate it via + `ConfigModel.model_validate(...)`. +2. Build the base Snakemake command from `config.profile.parsable()` plus + `--configfile` and `--cores`. +3. Append flags from CLI options: + - `--dry-run` adds `--dry-run`. + - `--unlock` adds `--unlock`. + - `--report FILE` adds `--report-after-run --report FILE` (only if not a + dry run). +4. Append the target rule name (e.g. `experiment_all`) and any extra Snakemake + args passed after `--`. +5. Write the full command to `.evalml_snakemake_cmd.txt` so the Snakefile's + `onstart:` hook can echo it back, then `subprocess.run(...)` it. + +`--dag` and `--rulegraph` short-circuit step 4: instead of running the +workflow, they ask Snakemake for a Graphviz `dot` representation, send it to +[kroki.io](https://kroki.io) for rendering, and write `dag.svg` / +`rulegraph.svg` next to your config. + +## Common option block + +All four subcommands share the same option set, defined by the +`workflow_options` decorator in `cli.py`: + +| Option | Default | Effect | +| --- | --- | --- | +| `--dry-run` / `-n` | off | Pass `--dry-run` to Snakemake; nothing is executed. | +| `--unlock` | off | Pass `--unlock` to release a stale Snakemake lock. | +| `--verbose` / `-v` | off | If unset, EvalML appends `--quiet rules` to dampen Snakemake output. | +| `--cores` / `-c` | `4` | Local cores Snakemake may use. SLURM jobs also respect `profile.jobs`. | +| `--report [FILE]` | none | Generate `_report.html` (or the file you specify). | +| `--dag` | off | Render the full DAG via kroki.io. | +| `--rulegraph` | off | Render only the rule graph via kroki.io. | +| `-- EXTRA_SMK_ARGS` | none | Anything after `--` is forwarded to Snakemake verbatim. | + +## Forcing re-runs + +EvalML decides what to rebuild based on file timestamps and the hash +machinery in `common.smk`. When you need to override that — typically +because a checkpoint mutated in place, or because you want to test a +new metric on already-computed inputs — pass Snakemake's force flags +after `--`: + +```bash +# Force EVERY rule to re-run, even if outputs exist. +evalml experiment config.yaml -- -F + +# Force one specific rule (and its downstream dependents) to re-run. +evalml experiment config.yaml -- -R verification_metrics + +# Force a rule for a specific wildcard combination via --until. +evalml experiment config.yaml -- --until verification_metrics +``` + +`-F` is `--forceall` (Snakemake), `-R` is `--forcerun`, and `--until` +stops execution after a named rule completes. These compose with the +other `-- …` forwards documented in the option table above. + +The most common use is `evalml experiment config.yaml -- -F` after a +checkpoint or baseline archive has been mutated in place — see the +warning in [Configuration → run identity](configuration.md#what-is-not-hashed-checkpoint-contents). + +## When to use which subcommand + +- **`experiment`** — full evaluation: data prep + inference + verification + + dashboard + plots. +- **`showcase`** — specific case-study visualisation: data prep + inference + visualisation (forecast animations, meteograms). +- **`sandbox`** — only build per-checkpoint inference sandboxes (squashfs + + zip), useful when handing models to collaborators. +- **`make`** — drop down to a specific Snakemake target. Use this when + debugging a single rule, e.g.: + ```bash + evalml make config.yaml inference_all + ``` diff --git a/docs/source/user_guide/configuration.md b/docs/source/user_guide/configuration.md new file mode 100644 index 00000000..6a6506f1 --- /dev/null +++ b/docs/source/user_guide/configuration.md @@ -0,0 +1,294 @@ +# Configuration + +Every EvalML run is driven by a YAML file that is validated against the +Pydantic models defined in [src/evalml/config.py](../../../src/evalml/config.py). +This page is the conceptual walkthrough; the full machine-readable schema is +in [reference/config_schema](../reference/config_schema.md), and the model +classes are auto-documented under [modules/evalml](../modules/evalml.md). + +## Top-level shape + +```yaml +description: ... +config_label: ... # optional; defaults to the YAML stem +dates: ... +runs: ... +truth: ... +stratification: ... +locations: ... +profile: ... +``` + +`extra: forbid` is set on `ConfigModel`, so misspelled keys at this level fail +validation immediately. + +## `dates` + +Two equivalent shapes are accepted, both producing a list of initialisation +times: + +```yaml +# Range form +dates: + start: 2020-01-01T00:00 + end: 2020-01-10T00:00 + frequency: 60h # validated by regex `^\d+[hd]$` +``` + +```yaml +# Explicit list form, useful for case studies +dates: + - 2020-01-01T00:00 + - 2020-01-03T12:00 + - 2020-02-14T06:00 +``` + +The range form is parsed into a list internally by `parse_reference_times` in +`workflow/rules/common.smk`. + +## `runs` + +`runs` is a heterogeneous list. Each item is a single-key mapping whose key is +one of `forecaster`, `interpolator` (temporal downscaler), or `baseline`, and whose value is the corresponding model. + +### Forecaster / Temporal downscaler + +```yaml +- forecaster: + checkpoint: https://mlflow.ecmwf.int/#/experiments/103/runs/d0846032fc... + label: M-1 forecaster + steps: 0/120/6 # start/end/step in hours + config: resources/inference/configs/sgm-forecaster-global.yaml + extra_requirements: + - git+https://github.com/ecmwf/anemoi-inference.git@0.8.3 + inference_resources: # optional + slurm_partition: long-shared + gpu: 2 + runtime: 1h + disable_local_eccodes_definitions: false +``` + +Field meanings: + +- **`checkpoint`** — MLflow URL, Hugging Face URL ending in `.ckpt`, or a + local path that exists. The URL host determines the retrieval mechanism + (see [Inference workflow](../workflow/inference.md)). +- **`label`** — the human-readable name shown on plots and the dashboard. + Excluded from the run's identity hash. +- **`steps`** — `start/end/step` in hours, validated by `validate_steps`. +- **`config`** — either a Python dict to override the inference config inline + or a path to a YAML template under `resources/inference/configs/`. If + omitted, defaults to `resources/inference/configs/forecaster.yaml` for + forecasters and `interpolator.yaml` for temporal downscaling. +- **`extra_requirements`** — additional pip-installable dependencies merged + into the auto-generated `requirements.txt` for the inference venv. +- **`inference_resources`** — overrides the SLURM defaults for this run only. +- **`disable_local_eccodes_definitions`** — skip pointing + `ECCODES_DEFINITION_PATH` at the venv-bundled COSMO definitions. + +`InterpolatorConfig` adds a single optional field: + +```yaml +- interpolator: + checkpoint: ... + config: resources/inference/configs/interpolator.yaml + forecaster: # nested ForecasterConfig + checkpoint: ... + steps: 0/120/6 + ... +``` + +The temporal downscaling pulls forecasts from `forecaster` instead of from analysis data. If `forecaster` is omitted, the temporal downscaling runs on analysis input. + +### Baseline + +```yaml +- baseline: + label: COSMO-E + root: /store_new/mch/msopr/ml/COSMO-E + steps: 10/120/1 +``` + +The `baseline_id` used in output paths is derived from `Path(root).stem`. The +top-level `baselines:` key is still accepted for backwards compatibility but +deprecated — prefer entries inside `runs:`. + +## `truth` + +```yaml +truth: + label: COSMO KENDA + root: /scratch/.../mch-co2-an-archive-0p02-2015-2020-6h-v3-pl13.zarr +``` + +If `root` ends in `peakweather`, EvalML will trigger +`data_download_obs_from_peakweather` to fetch station observations from +Hugging Face. + +## `stratification` + +```yaml +stratification: + regions: + - jura + - mittelland + - voralpen + root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 +``` + +Each entry is interpreted as `/.shp`. Verification scripts read +these shapefiles via `ShapefileSpatialAggregationMasks` and produce per-region +metrics in addition to the always-on `all` aggregate. + +## `thresholds` (optional) + +Adds categorical verification on top of the continuous metrics. Each entry +maps a parameter name to a dict of operator → list of threshold values: + +```yaml +thresholds: + TOT_PREC: + gt: [0.0, 0.001, 0.005] + U_10M: + gt: [2.5, 5.0, 10.0] + T_2M: + lt: [273.15] + gt: [288.15, 298.15] +``` + +Operator keys must be one of `gt`, `ge`, `lt`, `le`, `eq`, `ne` (validated by +a Pydantic `field_validator` on `ConfigModel.thresholds`). For each +`(operator, value)` pair, the verification pipeline builds a 2×2 contingency +table using +[`scores.categorical.ThresholdEventOperator`](https://scores.readthedocs.io/), +and stores it as a `contingency_table` variable on the per-init `verif.nc`, +keyed by a `threshold` dimension whose values are encoded as +`{op}_{value}` with the decimal point replaced by `p` +(`gt_0p001`, `lt_273p15`, …). Use +[`verification.decode_metric`](../modules/verification.md) to render those +labels back to human-readable form. + +If you omit the `thresholds:` block entirely, only the continuous metrics +are computed. + +## `dashboard` + +```yaml +dashboard: + stratification: + - season # group by JFM / AMJ / JAS / OND + # - region # also stratify by Stratification.regions + # - init_hour # also stratify by hour-of-day of init_time +``` + +`dashboard.stratification` is forwarded as the `--stratification` argument +to `report_experiment_dashboard.py` and controls which faceting axes the +dashboard exposes for browsing. Any of `season`, `region`, `init_hour` may +be enabled — list at least one. + +## `locations` + +```yaml +locations: + output_root: output/ +``` + +`output_root` becomes `OUT_ROOT` in the Snakefile and roots all +intermediate and final paths. + +## `profile` + +`profile` is forwarded to Snakemake via `Profile.parsable()`. Tune it to match +your executor: + +```yaml +profile: + executor: slurm + global_resources: + gpus: 16 # max concurrent GPUs across all jobs + default_resources: + slurm_partition: postproc + cpus_per_task: 1 + mem_mb_per_cpu: 1800 + runtime: 1h + jobs: 50 # max parallel job submissions + batch_rules: + plot_forecast_frame: 32 # group 32 invocations into one submission +``` + +`batch_rules` becomes a pair of `--groups` / `--group-components` arguments +to Snakemake, and is the simplest way to keep small plotting jobs from +flooding the scheduler. + +Do not confuse `profile.default_resources` with `inference_resources`. `profile.default_resources` are the workflow-wide defaults; a run's +`inference_resources` is a per-run override that only affects that run's +`inference_execute` invocations. Every field of `inference_resources` is +optional — anything you omit falls back to the hardcoded defaults in +`inference.smk` (`short-shared`, 24 CPUs, 8 GB/CPU, 40 min, 1 GPU), not to +`profile.default_resources`. The two are independent paths into Snakemake's +resource system. + +## Run identity: how `env_id` and `run_id` are derived + +EvalML is careful to *only* rebuild the inference environment when something +that genuinely affects the environment changes. The hashing logic lives in +[workflow/rules/common.smk](../../../workflow/rules/common.smk) and uses +two field sets exposed by `RunConfig`: + +| Constant | Fields | +| --- | --- | +| `RunConfig.ENV_FIELDS` | `checkpoint`, `extra_requirements`, `disable_local_eccodes_definitions` | +| `RunConfig.HASH_EXCLUDE` | `label`, `inference_resources` | + +From those: + +- **`env_id`** = `{model_type}-{model_id}-{env_hash}` (with `-on-{forecaster_env}` + appended for temporal downscaling). Determines which venv / squashfs is built. +- **`run_id`** = `{env_id}/{run_hash}`. Adds a hash of the inference config + YAML and `steps`. Determines where outputs land. + +Practical consequences: + +- Changing only the `label` does not invalidate any cache. +- Changing `steps` or the inference config creates a new run directory but + reuses the existing venv/squashfs. +- Changing the `checkpoint` field (i.e. the path or URL itself), or + `extra_requirements`, triggers a full environment rebuild. + +### What is *not* hashed: checkpoint contents + +```{warning} +Only the **string value** of the `checkpoint` field enters the hash — +its *contents* do not. If you mutate a checkpoint in place while keeping +the URL or local path the same, EvalML will reuse the cached +`inference-last.ckpt`, the cached venv/squashfs, and every downstream +output, because the hash hasn't changed. Force a rebuild with `-F` +(or `-R ` for a specific step) — see +{ref}`Iterate without re-running everything `. +``` + +For MLflow URLs this is rarely an issue (the run ID is content-addressed +upstream), but it bites with local checkpoint paths and pinned Hugging +Face files. + +### Per-run isolation + +Each entry in `runs:` gets its **own** inference environment — a +`uv`-built virtualenv snapshotted to `venv.squashfs` and mounted on the +compute node at runtime. This has two important consequences: + +- **Runs can pin different dependency versions.** Two forecasters in + the same experiment can require different `anemoi-inference` versions + (or any other extra dependency) without conflict; each one's + `extra_requirements` is merged with the deps recorded in its + checkpoint's MLflow metadata into a private `requirements.txt`. +- **The EvalML virtualenv (`.venv/`) does not contain + `anemoi-inference`.** EvalML drives the workflow; the inference + packages live exclusively inside each run's squashfs image and are + invoked there via `squashfs-mount … -- bash -c 'anemoi-inference run …'`. + This is why you don't need to install `anemoi-inference` to use the + CLI, and why an `anemoi-inference` upgrade is a per-run concern, not + a project-wide one. + +The full mechanism is documented in [Outputs and wildcards](outputs.md) and +[Inference workflow](../workflow/inference.md). diff --git a/docs/source/user_guide/examples.md b/docs/source/user_guide/examples.md new file mode 100644 index 00000000..ed13ba22 --- /dev/null +++ b/docs/source/user_guide/examples.md @@ -0,0 +1,121 @@ +# Example configs + +The `config/` directory ships eight ready-to-run experiment configurations, +covering the most common scenarios. They double as living tests: every +example is loaded by `tests/conftest.py` and validated against the schema. + +| File | Scenario | +| --- | --- | +| `forecasters-co2.yaml` | COSMO-2 forecaster comparison, 3 storm events | +| `forecasters-co2-disentangled.yaml` | "Disentangled" COSMO-2 forecaster variant | +| `forecasters-co1e.yaml` | COSMO-1E emulator fine-tuned on analysis (1 km, Jan 2020) | +| `forecasters-ich1.yaml` | ICON-CH1 single forecaster (stage_C, 1 km) | +| `forecasters-ich1-oper.yaml` | Operational ICON-CH1 configuration | +| `forecasters-ich1-oper-fixed.yaml` | Fixed operational ICON-CH1 variant | +| `interpolators-co2.yaml` | M-2 temporal downscaler on COSMO-2 | +| `interpolators-ich1.yaml` | ICON-CH1 temporal downscaler with multi-dataset support | + +## Picking a starting point + +If you are building a new evaluation: + +- **Comparing two forecasters** — copy `forecasters-co2.yaml` and replace the + two `forecaster:` entries with your checkpoints. Adjust `dates` and + `truth` as needed. +- **Evaluating a temporal downscaler** — copy `interpolators-co2.yaml`. Note the + `forecaster:` block nested inside `interpolator:`; remove it to run the + temporal downscaler on analysis input instead of forecaster output. +- **Operational ICON-CH1** — start from `forecasters-ich1-oper.yaml`. It uses + the operational inference config templates under `resources/inference/`. + +## Reading an example + +A condensed walkthrough of `forecasters-co2.yaml`: + +```yaml +description: | + COSMO-E forecaster emulator on three storm events. + +dates: + - 2020-02-09T00:00 + - 2021-01-13T12:00 + - 2023-08-25T00:00 + +runs: + - forecaster: + checkpoint: https://mlflow.ecmwf.int/.../d0846032fc7248a58b089cbe8fa4c511 + label: M-1 forecaster + steps: 0/120/6 + config: resources/inference/configs/sgm-forecaster-global.yaml + - baseline: + label: COSMO-E + root: /store_new/mch/msopr/ml/COSMO-E + steps: 10/120/1 + +truth: + label: COSMO KENDA + root: /scratch/.../mch-co2-an-archive-0p02-2015-2020-6h-v3-pl13.zarr + +stratification: + regions: + - jura + - mittelland + - voralpen + root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 + +thresholds: + TOT_PREC: + gt: [0.0, 0.001, 0.005] + T_2M: + lt: [273.15] + gt: [288.15, 298.15] + +dashboard: + stratification: + - season + +locations: + output_root: output/ + +profile: + executor: slurm + global_resources: { gpus: 16 } + default_resources: + slurm_partition: postproc + cpus_per_task: 1 + mem_mb_per_cpu: 1800 + runtime: 1h + jobs: 50 + batch_rules: + plot_forecast_frame: 32 +``` + +Things worth highlighting: + +- `dates` is in **explicit-list form** because the storms are hand-picked. +- The `forecaster:` `config` points at a bundled inference template under + `resources/inference/configs/`. Inline dict overrides are supported but + rarely needed. +- The `baseline:` entry uses `steps: 10/120/1`, which means the baseline + starts at lead time 10 h with hourly cadence; this is independent of the + forecaster's `0/120/6`. +- `batch_rules.plot_forecast_frame: 32` keeps SLURM happy when generating + hundreds of frames. +- `thresholds:` adds a 2×2 contingency table per parameter per threshold + to each `verif.nc`; the values land under the `contingency_table` + variable and feed the dashboard's categorical-skill panels. See + [Configuration → thresholds](configuration.md#thresholds-optional). +- `dashboard.stratification: [season]` activates the season facet in the + dashboard. Add `region` and/or `init_hour` to expose the other axes. + +## Schema validation in your editor + +All shipped examples include a YAML language-server hint: + +```yaml +# yaml-language-server: $schema=../workflow/tools/config.schema.json +``` + +If you copy an example into a new location, keep that comment so VSCode and +other YAML-aware editors can pull descriptions and autocompletion from the +generated schema. diff --git a/docs/source/user_guide/outputs.md b/docs/source/user_guide/outputs.md new file mode 100644 index 00000000..76062788 --- /dev/null +++ b/docs/source/user_guide/outputs.md @@ -0,0 +1,112 @@ +# Outputs and wildcards + +All workflow outputs are rooted at `OUT_ROOT`, which is taken from +`locations.output_root` in the config. Three top-level subdirectories live +underneath: + +```text +{OUT_ROOT}/ +├── data/ # intermediate artefacts +│ ├── runs/{env_id}/ # per-environment, shared across configs +│ │ ├── inference-last.ckpt +│ │ ├── requirements.txt +│ │ ├── venv.squashfs +│ │ └── {config_hash}/ # per-run-config artefacts +│ │ ├── summary.md +│ │ ├── verif_aggregated.nc +│ │ └── {init_time}/ # per-initialisation-time artefacts +│ │ ├── config.yaml +│ │ ├── grib/ +│ │ └── verif.nc +│ ├── baselines/{baseline_id}/ +│ │ └── {init_time}/verif.nc +│ └── observations/ # PeakWeather and other obs caches +├── logs/ # one sub-directory per rule +└── results/{experiment_name}/ # final products + ├── dashboard/ + └── plots/ +``` + +The split between `{env_id}/` and `{env_id}/{config_hash}/` reflects the +identity contract documented in [Configuration](configuration.md): expensive +artefacts (the venv and squashfs) live at the env level so they are reused +across runs that only differ in inference config. + +```{note} +`{run_id}` in the wildcard table below is exactly +`{env_id}/{config_hash}` — i.e. it spans **two** directory components, +not one. In the tree above, anything rooted at `data/runs/{env_id}/{config_hash}/` +is equivalent to `data/runs/{run_id}/`; the docs and the Snakemake rules +use both forms interchangeably. The +`wildcard_constraints: showcase=r"[^/]+"` block in the Snakefile exists +specifically because `{run_id}` contains a `/`. +``` + +(whats-inside-a-verif-nc)= +## What's inside a `verif.nc` + +Each per-init `verif.nc` is an `xarray.Dataset` keyed by `region` and +`parameter`. Continuous metrics (`BIAS`, `MSE`, `MAE`, `CORR`) and +statistics (`mean`, `var`, `min`, `max`) appear as scalar data variables +per region/parameter. When `thresholds:` is set in the config, an +additional `contingency_table` variable appears on a `threshold` +dimension; its values are encoded `{op}_{value}` (e.g. `gt_0p001` = +"forecast/obs > 0.001"). Use +[`verification.decode_metric`](../modules/verification.md) to render +the encoded labels back to human-readable form. + +## Wildcard reference + +| Wildcard | Format | Example | +| --- | --- | --- | +| `{env_id}` | `{type}-{model_id}-{env_hash}` (`-on-{forecaster_env}` for temporal downscalers) | `forecaster-1a2b-c3d4` | +| `{run_id}` | `{env_id}/{config_hash}` | `forecaster-1a2b-c3d4/e5f6` | +| `{baseline_id}` | `Path(root).stem` | `COSMO-E` | +| `{init_time}` | `%Y%m%d%H%M` | `202001011200` | +| `{experiment}` | `{YYYYMMDD}_{label}_{hash}` | `20260331_demo_a1b2` | +| `{param}` | variable name | `T_2M`, `TOT_PREC` | +| `{region}` | geographic region slug | `switzerland`, `globe` | +| `{leadtime}` | zero-padded hours | `000`, `006`, `024` | +| `{showcase}` | same as `{experiment}`; constrained to a single path component | `20260331_demo_a1b2` | + +The `wildcard_constraints: showcase=r"[^/]+"` block in the Snakefile is +deliberate: because `run_id` already contains a `/`, Snakemake would otherwise +greedily absorb part of `run_id` into `showcase` when matching paths like +`results/{showcase}/{run_id}/...`. + +## Logs and sentinel files + +Every rule declares a log file under: + +```text +{OUT_ROOT}/logs/{rule_name}/{wildcards}.log +``` + +When a rule produces a sentinel `.ok` file to mark successful completion, it +sits alongside the log: + +```text +{OUT_ROOT}/logs/{rule_name}/{wildcards}.ok +``` + +Multiple wildcards are joined with a hyphen: `{run_id}-{init_time}.log`. The +forecast / temporal downscaler preparation rules write a `.ok` so downstream rules can depend on the log, not on a directory whose timestamp is unstable. + +## What `experiment_name` means + +`experiment_name` (used as `{experiment}` and `{showcase}`) is built once per +invocation in the Snakefile: + +```python +EXPERIMENT_NAME = f"{WHEN}_{CONFIG_LABEL}_{CONFIG_HASH}" +``` + +- `WHEN` — today's date in `YYYYMMDD`. +- `CONFIG_LABEL` — `config_label` from the YAML, falling back to the YAML + filename stem. +- `CONFIG_HASH` — the master hash of every run's `env` and `run` hashes + (see `master_hash` in `common.smk`). + +This means re-running the same config on the same day reuses the same +`results/{experiment_name}/` directory, but a meaningful change to any run +produces a fresh directory rather than overwriting old results. diff --git a/docs/source/workflow/conventions.md b/docs/source/workflow/conventions.md new file mode 100644 index 00000000..65a4a90f --- /dev/null +++ b/docs/source/workflow/conventions.md @@ -0,0 +1,104 @@ +# Conventions + +These conventions apply to every rule and script under `workflow/`. They are +enforced informally during code review and (in part) by `snakefmt` and +`pre-commit`. + +## Rule names + +Rules use the pattern `{module}_{operation}[_{sub_operation}]` in snake_case. +The module prefix groups rules by their place in the pipeline: + +| Module | Rules prefix | Purpose | +| --- | --- | --- | +| `data.smk` | `data_` | Input data preparation | +| `inference.smk` | `inference_` | Checkpoint retrieval, environment setup, execution | +| `verification.smk` | `verification_` | Metrics calculation, aggregation, metric plots | +| `plot.smk` | `plot_` | Forecast visualisation (frames, animations, meteograms) | +| `report.smk` | `report_` | Dashboard and HTML report generation | + +Aggregate target rules use the pattern `{module}_all`: +`inference_all`, `verification_metrics_all`, etc. +Top-level entry points are named after the workflow mode: `experiment_all`, +`showcase_all`, `sandbox_all`. + +## Script names + +Scripts live in `workflow/scripts/` and mirror the rule that calls them: + +```text +{module}_{operation}.py # standard Python scripts +{module}_{operation}.mo.py # interactive Marimo notebooks +``` + +Examples: `inference_prepare.py`, `verification_metrics.py`, +`verification_aggregation.py`, `plot_forecast_frame.mo.py`. + +A script may be shared between multiple rules — `inference_prepare.py` is +used by both `inference_prepare_forecaster` and +`inference_prepare_interpolator`. When this happens, behaviour is selected +through the rule's `params:` block and read from `snakemake.params` inside +the script. + +## Log file paths + +Every rule declares a log file under: + +```text +{OUT_ROOT}/logs/{rule_name}/{wildcards}.log +``` + +Multiple wildcards are joined with a hyphen: `{run_id}-{init_time}.log`. + +When a rule produces a sentinel `.ok` file to mark successful completion, +it sits alongside the log: + +```text +{OUT_ROOT}/logs/{rule_name}/{wildcards}.ok +``` + +Sentinel files exist because Snakemake otherwise depends on the timestamp of +an output directory, which is unstable; the `.ok` is touched only on +success and is therefore a reliable trigger for downstream rules. + +## When to use `localrule: True` + +Mark a rule `localrule: True` when: + +- It runs a sub-second shell command (symlinking, file copy, JSON parsing). +- It only orchestrates other rules (e.g. `make_forecast_animation` calls + `convert`). +- It must run on the workflow head node for credentials reasons (MLflow + authentication is on the head node, not the SLURM compute nodes). + +Do *not* mark rules `localrule` if they do real work — `inference_execute`, +`verification_metrics`, and the plotting rules submit through SLURM via +`profile.executor`. + +## Resources + +When a rule submits to SLURM, declare: + +```python +resources: + slurm_partition="postproc", + cpus_per_task=24, + mem_mb=50_000, + runtime="60m", +``` + +For inference, the per-run override is plumbed through +`inference_resources` in the YAML config and resolved by `get_resource(wc, +field, default)` in `inference.smk`. + +## Adding a new rule + +1. Pick the right module (or create a new `.smk` if it's a wholly new + pipeline stage; remember to `include:` it in the Snakefile). +2. Name the rule `{module}_{operation}` and the script + `scripts/{module}_{operation}.py`. +3. Add a log path under `logs/{rule_name}/...`. +4. If it submits to SLURM, declare `resources` and avoid `localrule: True`. +5. Add a sentinel `.ok` file if the output is a directory. +6. Run `snakefmt workflow` to apply formatting; commit only after + `pre-commit run --all-files` is green. diff --git a/docs/source/workflow/data.md b/docs/source/workflow/data.md new file mode 100644 index 00000000..d8483e45 --- /dev/null +++ b/docs/source/workflow/data.md @@ -0,0 +1,82 @@ +# Data + +The `workflow/rules/data.smk` rule module owns one rule and a tiny piece of conditional logic at parse time: + +```python +if config["truth"]["root"].endswith("peakweather"): + output_peakweather_root = config["truth"]["root"] +else: + output_peakweather_root = OUT_ROOT / "data/observations/peakweather" +``` + +This means: if the user pointed `truth.root` at an existing PeakWeather +location, write directly there; otherwise cache observations under +`OUT_ROOT/data/observations/peakweather` so multiple experiments can share +the same download. + +## `data_download_obs_from_peakweather` + +```{eval-rst} +.. list-table:: + :widths: 20 80 + :header-rows: 0 + + * - **Local rule** + - yes — runs on the workflow head node; never submitted to SLURM. + * - **Output** + - ``output_peakweather_root`` (a directory) + * - **Run body** + - Instantiates ``peakweather.dataset.PeakWeatherDataset(root=output.root)``, + which downloads the dataset from Hugging Face on first use. +``` + +The rule has no `shell:` block; the entire download is implemented inside a +Snakemake `run:` block, which means PeakWeather is imported in the Snakemake +process itself. + +## How baseline data is consumed + +Baselines are not produced by a `data_*` rule; they are read directly by +`verification_metrics_baseline` from `{root}/FCST{YY}.zarr` (where `YY` is +the two-digit year extracted from `init_time`). The shape is determined by +the upstream archive and EvalML does not currently transform it before +verification. + +A standalone helper script `workflow/scripts/data_extract_baseline.py` exists +for ad-hoc baseline extraction, it is not invoked by any rule in the default pipeline and needs to be run manually. It supports both per-member extraction +(`--run_id 001`) and ensemble-mean aggregation (`--ensemble_mean`, which +loads every available member and averages them; in that case `--run_id` is +ignored). + +## Ground-truth dispatch + +`src/data_input/__init__.py` exposes a `load_truth_data(root, ...)` function +that dispatches based on `root`: + +- A path ending in `.zarr` is loaded via `load_analysis_data_from_zarr`. +- A path ending in `peakweather` (or pointing inside the PeakWeather cache) + is loaded via `load_obs_data_from_peakweather`. + +`load_forecast_data` does the analogous dispatch between GRIB directories +(EvalML inference output) and Zarr baselines. See the +[data_input module reference](../modules/data_input.md) for signatures. + +## Precipitation handling + +`TOT_PREC` requires extra care at every entry point: + +- **Analysis Zarr** (`load_analysis_data_from_zarr`): the upstream archive + stores precipitation in metres; the loader multiplies by 1000 to convert + to mm (the canonical unit used by every downstream metric and plot). +- **GRIB forecast** (`load_fct_data_from_grib`): expects cumulative-from- + start data (i.e. the `accumulate_from_start_of_forecast` post-processor + is enabled in anemoi-inference). The loader restricts to the requested + lead times, sets `lead_time=0` to 0 if it's missing or NaN, then + disaggregates with `.diff("lead_time")`. A sanity check raises if + `min(.diff())` is significantly negative — that would indicate the data + is already per-step accumulated, in which case a second disaggregation + would produce garbage. +- **Baseline Zarr** (`load_baseline_from_zarr`): the same disaggregation + logic as forecasts, plus the lead-time restriction is applied up-front + so sub-step baselines (hourly while the forecast cadence is 6-hourly) + produce the expected accumulation window. diff --git a/docs/source/workflow/inference.md b/docs/source/workflow/inference.md new file mode 100644 index 00000000..3246e627 --- /dev/null +++ b/docs/source/workflow/inference.md @@ -0,0 +1,140 @@ +# Inference + +`workflow/rules/inference.smk` covers four distinct concerns, in this order: + +1. **Resolve** the checkpoint by URI type (MLflow / Hugging Face / local). +2. **Build** a relocatable `uv` virtual environment from the checkpoint's + metadata and any `extra_requirements`, then snapshot it into a squashfs + image. +3. **Package** the venv + checkpoint + config + Jinja2 README into a + shareable sandbox zip. +4. **Execute** inference per (run_id, init_time) by mounting the squashfs + image with `squashfs-mount` and running `anemoi-inference run` over + `srun`. + +## Rule reference + +### `inference_get_checkpoint` + +| Property | Value | +| --- | --- | +| Local rule | yes | +| Outputs | `data/runs/{env_id}/inference-last.ckpt`, `data/runs/{env_id}/anemoi.json` | +| Log | `logs/inference_prepare_checkpoint/{env_id}.log` | + +The shell block branches on the checkpoint URI type: + +- **MLflow** (`mlflow.ecmwf.int`, `service.meteoswiss.ch`, + `servicedepl.meteoswiss.ch`) — symlinks the file resolved by + `scripts/inference_get_checkpoint_mlflow.py`. +- **Hugging Face** (`huggingface.co/...ckpt`) — extracts `repo_id` and + `file_path` from the URL with regex, then `cp $(uvx hf download ...)`. +- **Local path** — symlinks the path directly. + +After the checkpoint is in place, `anemoi-utils metadata --dump --json` +extracts the metadata blob into `anemoi.json`. + +### `inference_extract_requirements` + +Reads `anemoi.json` to recover the Python version and dependency list the +checkpoint was trained with, merges the user's `extra_requirements`, and +writes a `requirements.txt`. The merging logic lives in +`scripts/inference_extract_requirements.py`. + +### `inference_create_venv` + +Creates a relocatable virtualenv with `uv`: + +```bash +uv venv --managed-python --python $PYTHON_VERSION --relocatable --link-mode=copy {output.venv} +uv pip install -r {input.requirements} +python -m compileall -j 8 -o 0 -o 1 -o 2 .venv/lib/python*/site-packages +``` + +The compile pass produces `.pyc` files at all three optimisation levels so +import time stays low when the venv is mounted via squashfs. A final `import +eccodes` smoke-test ensures the GRIB stack is wired up before the venv is +snapshotted. + +The output is `temp(directory(...))`, so Snakemake removes the venv after +the squashfs is created. + +### `inference_make_squashfs_image` + +Wraps the venv into `data/runs/{env_id}/venv.squashfs`: + +```bash +mksquashfs $(realpath {input.venv}) {output.image} -no-recovery -noappend -Xcompression-level 3 +``` + +This is the artefact that `inference_execute` later mounts on the compute +nodes. + +### `inference_create_sandbox` + +Bundles the checkpoint, requirements, inference config, and a rendered +README (from `resources/inference/sandbox/readme.md.jinja2`) into a single +zip. The result is suitable for handing a checkpoint plus reproducible +runtime to another collaborator. + +### `inference_prepare_forecaster` / `inference_prepare_interpolator` + +Both rules call `scripts/inference_prepare.py`. They produce the per-init +`config.yaml`, a `resources/` directory of GRIB templates, and a `grib/` +output directory. The temporal downscaler ('interpolator') variant additionally: + +- Depends on the upstream forecaster's `inference_execute` `.ok` file, when + a `forecaster:` block is present. +- Symlinks the forecaster's output directory into `forecaster/` so the + temporal downscaler inference run can find it. +- Sets `params.forecaster_run_id`, used by the prepare script to wire the + upstream input. + +### `inference_execute` + +The actual run. It depends on the appropriate `inference_prepare_*` `.ok` +file (selected by `_inference_routing_fn`) and the squashfs image, and +launches: + +```bash +squashfs-mount {image}:/user-environment -- bash -c ' + source /user-environment/bin/activate + if [ "{disable_local_definitions}" = "False" ]; then + export ECCODES_DEFINITION_PATH=/user-environment/share/eccodes-cosmo-resources/definitions + fi + srun --partition=... --cpus-per-task=... ... anemoi-inference run config.yaml +' +``` + +Key points: + +- Resources (partition, CPUs, runtime, GPUs) are resolved per run via + `get_resource(wc, field, default)`, which falls back to sensible + defaults (`short-shared`, 24 CPUs, 8 GB/CPU, 40 min, 1 GPU). +- For multi-GPU runs (`gpus > 1`), `runner.parallel.cluster=slurm` is + appended to the inference command so anemoi-inference uses SLURM for + distributed coordination. +- Output is signalled by touching + `logs/inference_execute/{run_id}-{init_time}.ok`. Downstream rules depend + on the `.ok` rather than on the GRIB directory itself, which keeps DAG + invalidation predictable. + +## Identity contract + +`env_id` and `run_id` are computed in +[common.smk](../../../workflow/rules/common.smk) by `register_run`, +`env_entry_hash`, and `run_specific_hash`. The fields that go into each +hash are not free-form — they are explicitly listed in `RunConfig.ENV_FIELDS` +and consumed in `common.smk`: + +- `env_entry_hash` hashes only `ENV_FIELDS`. For temporal downscalers, it also + appends the upstream forecaster's `env_id` so a different upstream model + forces a new venv. +- `run_specific_hash` hashes `steps` plus the contents of the inference + config YAML. For temporal downscalers, it appends the forecaster's `run_id` so + the downscaler's outputs reflect *which* forecaster run it consumed. + +These hashes drive every output path under `data/runs/`, so a violation of +the contract (e.g. forgetting to include a new field in `ENV_FIELDS`) shows +up as silently reused environments. Tests in `tests/unit/test_run_identity.py` +guard the contract. diff --git a/docs/source/workflow/overview.md b/docs/source/workflow/overview.md new file mode 100644 index 00000000..03ca79ee --- /dev/null +++ b/docs/source/workflow/overview.md @@ -0,0 +1,97 @@ +# Workflow overview + +The Snakemake project lives entirely under `workflow/`. The CLI does not +contain any workflow logic — it just constructs a Snakemake invocation and +delegates. Reading this section will let you debug rules, add new ones, and +reason about why a particular file was (or was not) regenerated. + +## Snakefile structure + +The top of [workflow/Snakefile](../../../workflow/Snakefile) is small and +ordered deliberately: + +1. Validate the merged config through `ConfigModel` and write the result + back into Snakemake's `config` dict via `update_config(...)`. +2. Include the rule modules in dependency order: + - `rules/common.smk` — utilities, hashing, run/env registration. + - `rules/summary.smk` — human-readable run summary. + - `rules/data.smk` — observation data acquisition. + - `rules/inference.smk` — checkpoint retrieval, env build, inference. + - `rules/verification.smk` — metrics computation and aggregation. + - `rules/report.smk` — dashboard generation. + - `rules/plot.smk` — frames, animations, meteograms. +3. Read `.evalml_snakemake_cmd.txt` (written by the CLI) to surface the + invoked command in the `onstart:` banner. +4. Compute `EXPERIMENT_NAME = f"{WHEN}_{CONFIG_LABEL}_{CONFIG_HASH}"` and the + list of `CANDIDATES` (runs marked `_is_candidate=True`). +5. Define the wildcard constraint `showcase=r"[^/]+"` — necessary because + `run_id` contains `/`. +6. Define `onstart:` / `onsuccess:` / `onerror:` banners and the target rules. + +## Top-level target rules + +| Target | Triggered by | Inputs | +| --- | --- | --- | +| `experiment_all` | `evalml experiment` | `report_experiment_dashboard` + `verification_metrics_plot` for `EXPERIMENT_NAME` | +| `showcase_all` | `evalml showcase` | `make_forecast_animation` + `plot_meteogram` for selected params, regions, stations | +| `sandbox_all` | `evalml sandbox` | `inference_create_sandbox` outputs for all candidates | +| `inference_all` | `evalml make … inference_all` | Per-candidate, per-init-time `data/runs/{run_id}/{init_time}/raw` | +| `verification_metrics_all` | `evalml make …` | `verification_metrics` outputs for all candidates × init times | +| `verification_metrics_plot_all` | `evalml make …` | `verification_metrics_plot` for the current experiment | + +## Module map + +```text +workflow/ +├── Snakefile # entry point: includes, target rules, banners +├── rules/ +│ ├── common.smk # configuration parsing, hashing, registries +│ ├── summary.smk # write_summary +│ ├── data.smk # data_download_obs_from_peakweather +│ ├── inference.smk # inference_*, sandbox creation +│ ├── verification.smk # verification_metrics{,_baseline,_aggregation,_plot} +│ ├── report.smk # report_experiment_dashboard +│ └── plot.smk # plot_meteogram, plot_forecast_frame, animation +├── scripts/ +│ ├── data_*.py # called by data.smk +│ ├── inference_*.py # called by inference.smk +│ ├── verification_*.py # called by verification.smk +│ ├── report_*.py # called by report.smk +│ └── plot_*.mo.py # called by plot.smk (Marimo notebooks) +└── tools/ + └── config.schema.json # generated from src/evalml/config.py +``` + +Everything in `rules/` is imported by the Snakefile. Scripts in `scripts/` +are invoked through `script:` directives or `shell:` blocks; they import the +src-layout packages (`evalml`, `verification`, `data_input`, `plotting`) at +runtime. + +## How rules find each other + +`common.smk` populates several module-level dicts that all later rules read: + +- `RUN_CONFIGS` — every registered run, keyed by `run_id`. +- `ENV_CONFIGS` — unique inference environments, keyed by `env_id`. +- `BASELINE_CONFIGS` — baselines, keyed by `baseline_id`. +- `EXPERIMENT_PARTICIPANTS` — `dict[label, path-to-verif_aggregated.nc]`, + used to wire dashboard and metric plot inputs. +- `REGIONS` — comma-separated string of shapefile paths. +- `REFTIMES` — list of `datetime` objects. + +These are computed at parse time from `config["runs"]`, then kept stable +throughout the run. If you add a rule that needs to enumerate participants, +read from the registries rather than re-walking the YAML. + +## Reading the rest of this section + +- [Data](data.md) — how observations are pulled from PeakWeather and how + baseline data is wired into verification. +- [Inference](inference.md) — checkpoint retrieval, venv/squashfs build, + sandbox packaging, and `inference_execute`. +- [Verification](verification.md) — metric computation, aggregation, and + plotting. +- [Plotting](plotting.md) — Marimo-based frame and meteogram rules, + animation assembly. +- [Reporting](reporting.md) — the dashboard rule and its Jinja2 template. +- [Conventions](conventions.md) — naming, log paths, sentinel files. diff --git a/docs/source/workflow/plotting.md b/docs/source/workflow/plotting.md new file mode 100644 index 00000000..c1480b94 --- /dev/null +++ b/docs/source/workflow/plotting.md @@ -0,0 +1,101 @@ +# Plotting + +`workflow/rules/plot.smk` produces three kinds of artefact: + +- **Forecast frames** — single-time PNGs of a parameter over a region. +- **Forecast animations** — frames stitched into an animated GIF. +- **Meteograms** — per-station time series PNGs with forecast, truth, and + baselines overlaid. + +Both `plot_forecast_frame` and `plot_meteogram` shell out to **Marimo** +notebooks (`*.mo.py`) that can also be edited interactively, which is the +key reason the scripts use the `.mo.py` extension. + +## `plot_forecast_frame` + +| Property | Value | +| --- | --- | +| Output | `data/runs/{run_id}/{init_time}/frames/frame_{leadtime}_{param}_{region}.png` | +| Wildcard constraint | `leadtime=r"\d+"` | +| Resources | `slurm_partition=postproc`, 1 CPU, 10 min | + +The rule sets `ECCODES_DEFINITION_PATH` to the project venv's bundled COSMO +definitions, then runs: + +```bash +python scripts/plot_forecast_frame.mo.py \ + --input {grib_out_dir} \ + --date {init_time} \ + --outfn {output} \ + --param {param} \ + --leadtime {leadtime} \ + --region {region} \ + --accu {accu} +``` + +`--accu` is the forecast step in hours (derived from +`RUN_CONFIGS[run_id]["steps"].split("/")[2]`). It selects the right +accumulation-window colormap for `TOT_PREC` (`TOT_PREC_1H`, +`TOT_PREC_6H`, …) — see [Style and colormap choices](#style-and-colormap-choices) +below. + +Marimo notebooks accept the same CLI args, which means you can switch from a +batch render to an interactive edit by toggling the rule to `localrule: True` +and replacing `python` with `marimo edit`. The intended interactive command +is left as a comment in the rule body. + +## `make_forecast_animation` + +A localrule that takes the expanded list of frame outputs and runs: + +```bash +convert -delay {delay} -loop 0 {input} {output} +``` + +`{delay}` is computed as `10 * step` — i.e. forecasts with a 6-hourly cadence +get a 60 ms inter-frame delay. `convert` is part of ImageMagick. + +## `plot_meteogram` + +| Property | Value | +| --- | --- | +| Output | `results/{showcase}/{run_id}/{init_time}/{init_time}_{param}_{sta}.png` | +| Resources | `postproc`, 1 CPU, 10 min | + +Inputs: + +- The inference `.ok` for the forecast. +- `truth.root` for analysis. +- `data_download_obs_from_peakweather` output for station observations. + +The shell block builds a CLI argument list including any number of +baselines (collected by `_get_available_baselines(wc)`), then invokes the +Marimo notebook. As with frame plots, the rule has commented-out variants +that show how to launch `marimo edit` for live development. + +## Style and colormap choices + +The plotting helpers used by both notebooks live in `src/plotting/`: + +- `StatePlotter` — wraps an earthkit-plots GeoAxes; pre-computes a Delaunay + triangulation so successive `plot_field` calls on the same grid are cheap. + Includes a fast-path for orthographic projections (only the visible + hemisphere is triangulated). +- `DOMAINS` — named extents for `globe`, `europe`, `centraleurope`, + `switzerland`, with sensible default projections. +- `colormap_loader.load_ncl_colormap` — parses NCL `.ct` files from + `resources/report/plotting/`. +- `colormap_defaults.CMAP_DEFAULTS` — a `defaultdict` keyed by parameter + name (`T_2M`, `V_10M`, `TOT_PREC_1H`, `TOT_PREC_6H`, …) that returns + sensible `cmap`/`norm`/`units` for plotting. Note that precipitation is + keyed per accumulation window, so the rule passes `--accu` to let the + notebook pick the matching entry. + +If you add a new variable, prefer extending `CMAP_DEFAULTS` over hard-coding +colours in the notebook — the fallback returns viridis with a warning, +which is correct but ugly. + +The `showcase_all` target generates animations for `T_2M`, `SP_10M`, and +`TOT_PREC` over `globe`, `europe`, and `switzerland`. Extend the +`expand(...)` block in the Snakefile if you need additional parameters or +regions. diff --git a/docs/source/workflow/reporting.md b/docs/source/workflow/reporting.md new file mode 100644 index 00000000..7b22a9b5 --- /dev/null +++ b/docs/source/workflow/reporting.md @@ -0,0 +1,74 @@ +# Reporting + +`workflow/rules/report.smk` defines a single rule, +`report_experiment_dashboard`, that renders an interactive HTML dashboard +summarising one experiment. + +## `report_experiment_dashboard` + +| Property | Value | +| --- | --- | +| Local rule | yes | +| Output | `results/{experiment}/dashboard/` (directory; `htmlindex="dashboard.html"`) | +| Log | `logs/report_experiment_dashboard/{experiment}.log` | + +The rule consumes: + +- `EXPERIMENT_PARTICIPANTS.values()` — every aggregated `verif_aggregated.nc` + for the experiment (one per run + one per baseline). +- `resources/report/dashboard/template.html.jinja2` — the dashboard + template. +- `resources/report/dashboard/script.js` — the front-end JavaScript. +- The original config file (used to embed a copy in the dashboard). + +It then runs: + +```bash +python scripts/report_experiment_dashboard.py \ + --verif_files {verif} \ + --template {template} \ + --script {js_script} \ + --header_text "{header_text}" \ + --configfile "{configfile}" \ + --stratification {stratification} \ + --output {output} +``` + +`--stratification` is a space-separated list of dashboard facets drawn +from `config["dashboard"]["stratification"]`. The Pydantic `Dashboard` +model accepts any of `season`, `region`, `init_hour`; whatever's enabled +appears as selectable axes in the dashboard UI. + +`header_text` is computed by `make_header_text()` at parse time: + +- Explicit-list `dates` → + `"Explicit initializations from N runs have been used."` +- Range `dates` → + `"Verification against {truth} with initializations from {start} to {end} by {frequency}"` + +## Where dashboard logic lives + +The script in `workflow/scripts/report_experiment_dashboard.py` is +responsible for shaping the netCDF inputs into JSON suitable for the +front-end. The front-end JavaScript in +`resources/report/dashboard/script.js` then builds the interactive plots +(parameter/region/leadtime selectors, etc.). + +If you need to extend the dashboard, the typical change is: + +- Add a new metric in `verification.spatial.verify` so it lands in + `verif.nc`. +- Update `verification_aggregation.py` to keep the metric through the + aggregation step. +- Update `report_experiment_dashboard.py` to surface the metric in the + JSON payload. +- Update `template.html.jinja2` and/or `script.js` to render it. + +## Snakemake's HTML report + +This is distinct from the dashboard. When `evalml experiment --report` is +passed, EvalML appends `--report-after-run --report FILE` to Snakemake. +That instructs Snakemake itself to produce a self-contained HTML run +report — execution times, DAG, per-rule logs — independent of the EvalML +dashboard. It's useful for debugging slow rules and reviewing what was +executed. diff --git a/docs/source/workflow/verification.md b/docs/source/workflow/verification.md new file mode 100644 index 00000000..b5d25f34 --- /dev/null +++ b/docs/source/workflow/verification.md @@ -0,0 +1,112 @@ +# Verification + +`workflow/rules/verification.smk` turns inference outputs (per-init GRIB +directories) and baselines (Zarr) into per-init `verif.nc` files, then +aggregates and plots them. The metric implementations live in +[src/verification/spatial.py](../../../src/verification/spatial.py); see the +[verification module reference](../modules/verification.md) for the +docstrings. + +## The four-rule pipeline + +```text + verification_metrics verification_metrics_baseline + (per run, per init) (per baseline, per init) + │ │ + ▼ ▼ + verification_metrics_aggregation verification_metrics_aggregation_baseline + │ │ + └──────────────┬───────────────┘ + ▼ + verification_metrics_plot +``` + +### `verification_metrics` + +Inputs: the inference `.ok` file plus `truth.root`. Output: +`data/runs/{run_id}/{init_time}/verif.nc`. + +The shell command is: + +```bash +uv run scripts/verification_metrics.py \ + --forecast {grib_out_dir} \ + --truth {truth} \ + --reftime {init_time} \ + --steps {fcst_steps} \ + --label {fcst_label} \ + --truth_label {truth_label} \ + --regions {regions} \ + --threshold_dict "{threshold_dict}" \ + --output {output} +``` + +`uv run` is used so the script picks up the project's environment without +requiring a manual activation step. + +`--threshold_dict` is the literal repr of `config["thresholds"]` and may be +the empty dict, in which case only continuous metrics are computed. + +### `verification_metrics_baseline` + +Same script and arguments, but the input is a baseline Zarr expanded to +`{root}/FCST{YY}.zarr`. Output: +`data/baselines/{baseline_id}/{init_time}/verif.nc`. + +### `verification_metrics_aggregation` + +Calls `scripts/verification_aggregation.py` over all per-init `verif.nc` +files for one `run_id` (filtered through `_restrict_reftimes_to_hours`, +which currently passes everything through unchanged but exists as a hook +for hour-of-day stratification). Output: +`data/runs/{run_id}/verif_aggregated.nc`. + +`verification_metrics_aggregation_baseline` is defined via Snakemake's +`use rule … with:` syntax, reusing the same body but pointing at baseline +inputs and outputs. + +### `verification_metrics_plot` + +Takes every aggregated `verif_aggregated.nc` for the experiment +(`EXPERIMENT_PARTICIPANTS.values()`) and produces a directory of PNGs at +`results/{experiment}/plots/`. The directory is wrapped in a +`report(directory(...), patterns=["{name}.png"])` so the rendered HTML +report can include the figures. + +## Metric computation + +The heavy lifting is in `verification.verify(...)`: + +- Computes per-parameter continuous scores (BIAS, MSE, MAE, CORR) via the + [`scores`](https://scores.readthedocs.io/) library (≥ 2.0). +- Computes per-parameter statistics (mean, var, min, max). +- Aggregates per region, including a hardcoded `all` region. +- When `threshold_dict` is provided, also computes 2×2 contingency tables + per `(parameter, operator, threshold)` triple via + `_binary_confusion_matrix` and `scores.categorical.ThresholdEventOperator`. + The result is stored as a `contingency_table` variable on a `threshold` + dimension whose values are encoded as `{op}_{value}` (e.g. `gt_0p001`). +- Optionally runs in parallel via Dask, given a `num_workers`. + +The CORR metric still uses `xr.corr` under the hood for backwards +compatibility, and `R²` / `VAR` are no longer emitted (they were derivable +from CORR and the statistics dataset). To translate an encoded threshold +label back into human-readable form, use +[`verification.decode_metric`](../modules/verification.md). + +For coordinate alignment between forecast and truth on different grids, +`map_forecast_to_truth(fcst, truth)` first does a spherical nearest-neighbour +mapping (`spherical_nearest_neighbor_indices`) so we don't suffer from the +distortions that come with naive lat/lon Euclidean distance. + +## Region masks + +Region masks are produced by `ShapefileSpatialAggregationMasks` in the same +module. It takes: + +- `shp` — a single shapefile or a list of shapefiles. +- `src_crs` and `dst_crs` — the source and destination CRS. + +It exposes `get_masks(lat, lon)` which returns an `xarray.DataArray` with a +`region` dimension. The `all` region is always present and covers the whole +grid; the named regions correspond one-to-one with the shapefiles passed in. diff --git a/pyproject.toml b/pyproject.toml index 3bbf08e9..b668485a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,15 @@ dev = [ "pre-commit>=4.2.0", "snakefmt>=0.11.0", ] +docs = [ + "sphinx>=7.4", + "sphinx-rtd-theme>=2.0", + "myst-parser>=3.0", + "sphinx-autobuild>=2024.10.3", + "sphinx-copybutton>=0.5.2", + "sphinx-click>=6.0", + "linkify-it-py>=2.0", +] [tool.pytest.ini_options] markers = [ diff --git a/src/plotting/colormap_loader.py b/src/plotting/colormap_loader.py index 348f472c..7881610f 100644 --- a/src/plotting/colormap_loader.py +++ b/src/plotting/colormap_loader.py @@ -14,10 +14,9 @@ def load_ncl_colormap(filename): Returns ------- dict - Dictionary containing the colormap and normalisation generated from the - colormap file - {cmap : matplotlib.colors.ListedColormap, - norm : matplotlib.colors.BoundaryNorm } + Dictionary with keys ``cmap`` (``matplotlib.colors.ListedColormap``), + ``norm`` (``matplotlib.colors.BoundaryNorm``), and ``bounds`` + (the list of boundary values). """ cmap_path = BASE_DIR / filename if not cmap_path.exists(): diff --git a/uv.lock b/uv.lock index 892808d8..37ada6cc 100644 --- a/uv.lock +++ b/uv.lock @@ -20,6 +20,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/53/1c/8feedd607cc14c5df9aef74fe3af9a99bf660743b842a9b5b1865326b4aa/adjustText-1.3.0-py3-none-any.whl", hash = "sha256:da23d7b24b6db5ffa039bb136bfa556207365e32f48ac74b07ad26dd485bc691", size = 13154, upload-time = "2024-10-31T16:45:35.227Z" }, ] +[[package]] +name = "alabaster" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, +] + [[package]] name = "alembic" version = "1.16.5" @@ -180,6 +189,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, ] +[[package]] +name = "babel" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, +] + [[package]] name = "black" version = "24.10.0" @@ -1107,6 +1125,16 @@ dev = [ { name = "pre-commit" }, { name = "snakefmt" }, ] +docs = [ + { name = "linkify-it-py" }, + { name = "myst-parser" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "sphinx-autobuild" }, + { name = "sphinx-click" }, + { name = "sphinx-copybutton" }, + { name = "sphinx-rtd-theme" }, +] [package.metadata] requires-dist = [ @@ -1140,6 +1168,15 @@ dev = [ { name = "pre-commit", specifier = ">=4.2.0" }, { name = "snakefmt", specifier = ">=0.11.0" }, ] +docs = [ + { name = "linkify-it-py", specifier = ">=2.0" }, + { name = "myst-parser", specifier = ">=3.0" }, + { name = "sphinx", specifier = ">=7.4" }, + { name = "sphinx-autobuild", specifier = ">=2024.10.3" }, + { name = "sphinx-click", specifier = ">=6.0" }, + { name = "sphinx-copybutton", specifier = ">=0.5.2" }, + { name = "sphinx-rtd-theme", specifier = ">=2.0" }, +] [[package]] name = "execnet" @@ -1552,6 +1589,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] +[[package]] +name = "imagesize" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/e6/7bf14eeb8f8b7251141944835abd42eb20a658d89084b7e1f3e5fe394090/imagesize-2.0.0.tar.gz", hash = "sha256:8e8358c4a05c304f1fccf7ff96f036e7243a189e9e42e90851993c558cfe9ee3", size = 1773045, upload-time = "2026-03-03T14:18:29.941Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl", hash = "sha256:5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96", size = 9441, upload-time = "2026-03-03T14:18:27.892Z" }, +] + [[package]] name = "immutables" version = "0.21" @@ -1794,6 +1840,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/e9/0d4add7873a73e462aeb45c036a2dead2562b825aa46ba326727b3f31016/kiwisolver-1.4.9-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:fb940820c63a9590d31d88b815e7a3aa5915cad3ce735ab45f0c730b39547de1", size = 73929, upload-time = "2025-08-10T21:27:48.236Z" }, ] +[[package]] +name = "linkify-it-py" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "uc-micro-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/c9/06ea13676ef354f0af6169587ae292d3e2406e212876a413bf9eece4eb23/linkify_it_py-2.1.0.tar.gz", hash = "sha256:43360231720999c10e9328dc3691160e27a718e280673d444c38d7d3aaa3b98b", size = 29158, upload-time = "2026-03-01T07:48:47.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, +] + [[package]] name = "locket" version = "1.0.0" @@ -2101,6 +2159,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/53/8d8fa0ea32a8c8239e04d022f6c059ee5e1b77517769feccd50f1df43d6d/matplotlib-3.10.6-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d6ca6ef03dfd269f4ead566ec6f3fb9becf8dab146fb999022ed85ee9f6b3eb", size = 8693933, upload-time = "2025-08-30T00:14:22.942Z" }, ] +[[package]] +name = "mdit-py-plugins" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -2276,6 +2346,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "myst-parser" +version = "5.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "jinja2" }, + { name = "markdown-it-py" }, + { name = "mdit-py-plugins" }, + { name = "pyyaml" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/fa/7b45eef11b7971f0beb29d27b7bfe0d747d063aa29e170d9edd004733c8a/myst_parser-5.0.0.tar.gz", hash = "sha256:f6f231452c56e8baa662cc352c548158f6a16fcbd6e3800fc594978002b94f3a", size = 98535, upload-time = "2026-01-15T09:08:18.036Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/ac/686789b9145413f1a61878c407210e41bfdb097976864e0913078b24098c/myst_parser-5.0.0-py3-none-any.whl", hash = "sha256:ab31e516024918296e169139072b81592336f2fef55b8986aa31c9f04b5f7211", size = 84533, upload-time = "2026-01-15T09:08:16.788Z" }, +] + [[package]] name = "narwhals" version = "2.6.0" @@ -3358,6 +3446,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/31/f6/5fc0574af5379606ffd57a4b68ed88f9b415eb222047fe023aefcc00a648/rich_argparse-1.7.1-py3-none-any.whl", hash = "sha256:a8650b42e4a4ff72127837632fba6b7da40784842f08d7395eb67a9cbd7b4bf9", size = 25357, upload-time = "2025-05-25T20:20:33.793Z" }, ] +[[package]] +name = "roman-numerals" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, +] + [[package]] name = "rpds-py" version = "0.27.1" @@ -3867,6 +3964,205 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "snowballstemmer" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/a7/9810d872919697c9d01295633f5d574fb416d47e535f258272ca1f01f447/snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895", size = 105575, upload-time = "2025-05-09T16:34:51.843Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064", size = 103274, upload-time = "2025-05-09T16:34:50.371Z" }, +] + +[[package]] +name = "sphinx" +version = "9.0.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12'", +] +dependencies = [ + { name = "alabaster", marker = "python_full_version < '3.12'" }, + { name = "babel", marker = "python_full_version < '3.12'" }, + { name = "colorama", marker = "python_full_version < '3.12' and sys_platform == 'win32'" }, + { name = "docutils", marker = "python_full_version < '3.12'" }, + { name = "imagesize", marker = "python_full_version < '3.12'" }, + { name = "jinja2", marker = "python_full_version < '3.12'" }, + { name = "packaging", marker = "python_full_version < '3.12'" }, + { name = "pygments", marker = "python_full_version < '3.12'" }, + { name = "requests", marker = "python_full_version < '3.12'" }, + { name = "roman-numerals", marker = "python_full_version < '3.12'" }, + { name = "snowballstemmer", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/3f/4bbd76424c393caead2e1eb89777f575dee5c8653e2d4b6afd7a564f5974/sphinx-9.0.4-py3-none-any.whl", hash = "sha256:5bebc595a5e943ea248b99c13814c1c5e10b3ece718976824ffa7959ff95fffb", size = 3917713, upload-time = "2025-12-04T07:45:24.944Z" }, +] + +[[package]] +name = "sphinx" +version = "9.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", +] +dependencies = [ + { name = "alabaster", marker = "python_full_version >= '3.12'" }, + { name = "babel", marker = "python_full_version >= '3.12'" }, + { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, + { name = "docutils", marker = "python_full_version >= '3.12'" }, + { name = "imagesize", marker = "python_full_version >= '3.12'" }, + { name = "jinja2", marker = "python_full_version >= '3.12'" }, + { name = "packaging", marker = "python_full_version >= '3.12'" }, + { name = "pygments", marker = "python_full_version >= '3.12'" }, + { name = "requests", marker = "python_full_version >= '3.12'" }, + { name = "roman-numerals", marker = "python_full_version >= '3.12'" }, + { name = "snowballstemmer", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl", hash = "sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978", size = 3921742, upload-time = "2025-12-31T15:09:25.561Z" }, +] + +[[package]] +name = "sphinx-autobuild" +version = "2025.8.25" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "starlette" }, + { name = "uvicorn" }, + { name = "watchfiles" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/3c/a59a3a453d4133777f7ed2e83c80b7dc817d43c74b74298ca0af869662ad/sphinx_autobuild-2025.8.25.tar.gz", hash = "sha256:9cf5aab32853c8c31af572e4fecdc09c997e2b8be5a07daf2a389e270e85b213", size = 15200, upload-time = "2025-08-25T18:44:55.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/20/56411b52f917696995f5ad27d2ea7e9492c84a043c5b49a3a3173573cd93/sphinx_autobuild-2025.8.25-py3-none-any.whl", hash = "sha256:b750ac7d5a18603e4665294323fd20f6dcc0a984117026d1986704fa68f0379a", size = 12535, upload-time = "2025-08-25T18:44:54.164Z" }, +] + +[[package]] +name = "sphinx-click" +version = "6.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "docutils" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/ed/a9767cd1b8b7fbdf260a89d5c8c86e20e3536b9878579e5ab7965a291e55/sphinx_click-6.2.0.tar.gz", hash = "sha256:fc78b4154a4e5159462e36de55b8643747da6cda86b3b52a8bb62289e603776c", size = 27035, upload-time = "2025-12-04T19:33:05.437Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/bd/cb244695f67f77b0a36200ce1670fc42a6fe2770847e870daab99cc2b177/sphinx_click-6.2.0-py3-none-any.whl", hash = "sha256:1fb1851cb4f2c286d43cbcd57f55db6ef5a8d208bfc3370f19adde232e5803d7", size = 8939, upload-time = "2025-12-04T19:33:04.037Z" }, +] + +[[package]] +name = "sphinx-copybutton" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/2b/a964715e7f5295f77509e59309959f4125122d648f86b4fe7d70ca1d882c/sphinx-copybutton-0.5.2.tar.gz", hash = "sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd", size = 23039, upload-time = "2023-04-14T08:10:22.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, +] + +[[package]] +name = "sphinx-rtd-theme" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-jquery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/68/a1bfbf38c0f7bccc9b10bbf76b94606f64acb1552ae394f0b8285bfaea25/sphinx_rtd_theme-3.1.0.tar.gz", hash = "sha256:b44276f2c276e909239a4f6c955aa667aaafeb78597923b1c60babc76db78e4c", size = 7620915, upload-time = "2026-01-12T16:03:31.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/c7/b5c8015d823bfda1a346adb2c634a2101d50bb75d421eb6dcb31acd25ebc/sphinx_rtd_theme-3.1.0-py2.py3-none-any.whl", hash = "sha256:1785824ae8e6632060490f67cf3a72d404a85d2d9fc26bce3619944de5682b89", size = 7655617, upload-time = "2026-01-12T16:03:28.101Z" }, +] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, +] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, +] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, +] + +[[package]] +name = "sphinxcontrib-jquery" +version = "4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/f3/aa67467e051df70a6330fe7770894b3e4f09436dea6881ae0b4f3d87cad8/sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a", size = 122331, upload-time = "2023-03-14T15:01:01.944Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/85/749bd22d1a68db7291c89e2ebca53f4306c3f205853cf31e9de279034c3c/sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae", size = 121104, upload-time = "2023-03-14T15:01:00.356Z" }, +] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, +] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, +] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.43" @@ -4031,6 +4327,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, ] +[[package]] +name = "uc-micro-py" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/78/67/9a363818028526e2d4579334460df777115bdec1bb77c08f9db88f6389f2/uc_micro_py-2.0.0.tar.gz", hash = "sha256:c53691e495c8db60e16ffc4861a35469b0ba0821fe409a8a7a0a71864d33a811", size = 6611, upload-time = "2026-03-01T06:31:27.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/73/d21edf5b204d1467e06500080a50f79d49ef2b997c79123a536d4a17d97c/uc_micro_py-2.0.0-py3-none-any.whl", hash = "sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c", size = 6383, upload-time = "2026-03-01T06:31:26.257Z" }, +] + [[package]] name = "ujson" version = "5.11.0" @@ -4145,6 +4450,93 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8d/57/a27182528c90ef38d82b636a11f606b0cbb0e17588ed205435f8affe3368/waitress-3.0.2-py3-none-any.whl", hash = "sha256:c56d67fd6e87c2ee598b76abdd4e96cfad1f24cacdea5078d382b1f9d7b5ed2e", size = 56232, upload-time = "2024-11-16T20:02:33.858Z" }, ] +[[package]] +name = "watchfiles" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/f8/2c5f479fb531ce2f0564eda479faecf253d886b1ab3630a39b7bf7362d46/watchfiles-1.1.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f57b396167a2565a4e8b5e56a5a1c537571733992b226f4f1197d79e94cf0ae5", size = 406529, upload-time = "2025-10-14T15:04:32.899Z" }, + { url = "https://files.pythonhosted.org/packages/fe/cd/f515660b1f32f65df671ddf6f85bfaca621aee177712874dc30a97397977/watchfiles-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:421e29339983e1bebc281fab40d812742268ad057db4aee8c4d2bce0af43b741", size = 394384, upload-time = "2025-10-14T15:04:33.761Z" }, + { url = "https://files.pythonhosted.org/packages/7b/c3/28b7dc99733eab43fca2d10f55c86e03bd6ab11ca31b802abac26b23d161/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e43d39a741e972bab5d8100b5cdacf69db64e34eb19b6e9af162bccf63c5cc6", size = 448789, upload-time = "2025-10-14T15:04:34.679Z" }, + { url = "https://files.pythonhosted.org/packages/4a/24/33e71113b320030011c8e4316ccca04194bf0cbbaeee207f00cbc7d6b9f5/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f537afb3276d12814082a2e9b242bdcf416c2e8fd9f799a737990a1dbe906e5b", size = 460521, upload-time = "2025-10-14T15:04:35.963Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c3/3c9a55f255aa57b91579ae9e98c88704955fa9dac3e5614fb378291155df/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2cd9e04277e756a2e2d2543d65d1e2166d6fd4c9b183f8808634fda23f17b14", size = 488722, upload-time = "2025-10-14T15:04:37.091Z" }, + { url = "https://files.pythonhosted.org/packages/49/36/506447b73eb46c120169dc1717fe2eff07c234bb3232a7200b5f5bd816e9/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f3f58818dc0b07f7d9aa7fe9eb1037aecb9700e63e1f6acfed13e9fef648f5d", size = 596088, upload-time = "2025-10-14T15:04:38.39Z" }, + { url = "https://files.pythonhosted.org/packages/82/ab/5f39e752a9838ec4d52e9b87c1e80f1ee3ccdbe92e183c15b6577ab9de16/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bb9f66367023ae783551042d31b1d7fd422e8289eedd91f26754a66f44d5cff", size = 472923, upload-time = "2025-10-14T15:04:39.666Z" }, + { url = "https://files.pythonhosted.org/packages/af/b9/a419292f05e302dea372fa7e6fda5178a92998411f8581b9830d28fb9edb/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aebfd0861a83e6c3d1110b78ad54704486555246e542be3e2bb94195eabb2606", size = 456080, upload-time = "2025-10-14T15:04:40.643Z" }, + { url = "https://files.pythonhosted.org/packages/b0/c3/d5932fd62bde1a30c36e10c409dc5d54506726f08cb3e1d8d0ba5e2bc8db/watchfiles-1.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5fac835b4ab3c6487b5dbad78c4b3724e26bcc468e886f8ba8cc4306f68f6701", size = 629432, upload-time = "2025-10-14T15:04:41.789Z" }, + { url = "https://files.pythonhosted.org/packages/f7/77/16bddd9779fafb795f1a94319dc965209c5641db5bf1edbbccace6d1b3c0/watchfiles-1.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:399600947b170270e80134ac854e21b3ccdefa11a9529a3decc1327088180f10", size = 623046, upload-time = "2025-10-14T15:04:42.718Z" }, + { url = "https://files.pythonhosted.org/packages/46/ef/f2ecb9a0f342b4bfad13a2787155c6ee7ce792140eac63a34676a2feeef2/watchfiles-1.1.1-cp311-cp311-win32.whl", hash = "sha256:de6da501c883f58ad50db3a32ad397b09ad29865b5f26f64c24d3e3281685849", size = 271473, upload-time = "2025-10-14T15:04:43.624Z" }, + { url = "https://files.pythonhosted.org/packages/94/bc/f42d71125f19731ea435c3948cad148d31a64fccde3867e5ba4edee901f9/watchfiles-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:35c53bd62a0b885bf653ebf6b700d1bf05debb78ad9292cf2a942b23513dc4c4", size = 287598, upload-time = "2025-10-14T15:04:44.516Z" }, + { url = "https://files.pythonhosted.org/packages/57/c9/a30f897351f95bbbfb6abcadafbaca711ce1162f4db95fc908c98a9165f3/watchfiles-1.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:57ca5281a8b5e27593cb7d82c2ac927ad88a96ed406aa446f6344e4328208e9e", size = 277210, upload-time = "2025-10-14T15:04:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/74/d5/f039e7e3c639d9b1d09b07ea412a6806d38123f0508e5f9b48a87b0a76cc/watchfiles-1.1.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:8c89f9f2f740a6b7dcc753140dd5e1ab9215966f7a3530d0c0705c83b401bd7d", size = 404745, upload-time = "2025-10-14T15:04:46.731Z" }, + { url = "https://files.pythonhosted.org/packages/a5/96/a881a13aa1349827490dab2d363c8039527060cfcc2c92cc6d13d1b1049e/watchfiles-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bd404be08018c37350f0d6e34676bd1e2889990117a2b90070b3007f172d0610", size = 391769, upload-time = "2025-10-14T15:04:48.003Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5b/d3b460364aeb8da471c1989238ea0e56bec24b6042a68046adf3d9ddb01c/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8526e8f916bb5b9a0a777c8317c23ce65de259422bba5b31325a6fa6029d33af", size = 449374, upload-time = "2025-10-14T15:04:49.179Z" }, + { url = "https://files.pythonhosted.org/packages/b9/44/5769cb62d4ed055cb17417c0a109a92f007114a4e07f30812a73a4efdb11/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2edc3553362b1c38d9f06242416a5d8e9fe235c204a4072e988ce2e5bb1f69f6", size = 459485, upload-time = "2025-10-14T15:04:50.155Z" }, + { url = "https://files.pythonhosted.org/packages/19/0c/286b6301ded2eccd4ffd0041a1b726afda999926cf720aab63adb68a1e36/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30f7da3fb3f2844259cba4720c3fc7138eb0f7b659c38f3bfa65084c7fc7abce", size = 488813, upload-time = "2025-10-14T15:04:51.059Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2b/8530ed41112dd4a22f4dcfdb5ccf6a1baad1ff6eed8dc5a5f09e7e8c41c7/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8979280bdafff686ba5e4d8f97840f929a87ed9cdf133cbbd42f7766774d2aa", size = 594816, upload-time = "2025-10-14T15:04:52.031Z" }, + { url = "https://files.pythonhosted.org/packages/ce/d2/f5f9fb49489f184f18470d4f99f4e862a4b3e9ac2865688eb2099e3d837a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcc5c24523771db3a294c77d94771abcfcb82a0e0ee8efd910c37c59ec1b31bb", size = 475186, upload-time = "2025-10-14T15:04:53.064Z" }, + { url = "https://files.pythonhosted.org/packages/cf/68/5707da262a119fb06fbe214d82dd1fe4a6f4af32d2d14de368d0349eb52a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db5d7ae38ff20153d542460752ff397fcf5c96090c1230803713cf3147a6803", size = 456812, upload-time = "2025-10-14T15:04:55.174Z" }, + { url = "https://files.pythonhosted.org/packages/66/ab/3cbb8756323e8f9b6f9acb9ef4ec26d42b2109bce830cc1f3468df20511d/watchfiles-1.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:28475ddbde92df1874b6c5c8aaeb24ad5be47a11f87cde5a28ef3835932e3e94", size = 630196, upload-time = "2025-10-14T15:04:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/78/46/7152ec29b8335f80167928944a94955015a345440f524d2dfe63fc2f437b/watchfiles-1.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:36193ed342f5b9842edd3532729a2ad55c4160ffcfa3700e0d54be496b70dd43", size = 622657, upload-time = "2025-10-14T15:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/0a/bf/95895e78dd75efe9a7f31733607f384b42eb5feb54bd2eb6ed57cc2e94f4/watchfiles-1.1.1-cp312-cp312-win32.whl", hash = "sha256:859e43a1951717cc8de7f4c77674a6d389b106361585951d9e69572823f311d9", size = 272042, upload-time = "2025-10-14T15:04:59.046Z" }, + { url = "https://files.pythonhosted.org/packages/87/0a/90eb755f568de2688cb220171c4191df932232c20946966c27a59c400850/watchfiles-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:91d4c9a823a8c987cce8fa2690923b069966dabb196dd8d137ea2cede885fde9", size = 288410, upload-time = "2025-10-14T15:05:00.081Z" }, + { url = "https://files.pythonhosted.org/packages/36/76/f322701530586922fbd6723c4f91ace21364924822a8772c549483abed13/watchfiles-1.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:a625815d4a2bdca61953dbba5a39d60164451ef34c88d751f6c368c3ea73d404", size = 278209, upload-time = "2025-10-14T15:05:01.168Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f4/f750b29225fe77139f7ae5de89d4949f5a99f934c65a1f1c0b248f26f747/watchfiles-1.1.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:130e4876309e8686a5e37dba7d5e9bc77e6ed908266996ca26572437a5271e18", size = 404321, upload-time = "2025-10-14T15:05:02.063Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/f07a295cde762644aa4c4bb0f88921d2d141af45e735b965fb2e87858328/watchfiles-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5f3bde70f157f84ece3765b42b4a52c6ac1a50334903c6eaf765362f6ccca88a", size = 391783, upload-time = "2025-10-14T15:05:03.052Z" }, + { url = "https://files.pythonhosted.org/packages/bc/11/fc2502457e0bea39a5c958d86d2cb69e407a4d00b85735ca724bfa6e0d1a/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e0b1fe858430fc0251737ef3824c54027bedb8c37c38114488b8e131cf8219", size = 449279, upload-time = "2025-10-14T15:05:04.004Z" }, + { url = "https://files.pythonhosted.org/packages/e3/1f/d66bc15ea0b728df3ed96a539c777acfcad0eb78555ad9efcaa1274688f0/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f27db948078f3823a6bb3b465180db8ebecf26dd5dae6f6180bd87383b6b4428", size = 459405, upload-time = "2025-10-14T15:05:04.942Z" }, + { url = "https://files.pythonhosted.org/packages/be/90/9f4a65c0aec3ccf032703e6db02d89a157462fbb2cf20dd415128251cac0/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:059098c3a429f62fc98e8ec62b982230ef2c8df68c79e826e37b895bc359a9c0", size = 488976, upload-time = "2025-10-14T15:05:05.905Z" }, + { url = "https://files.pythonhosted.org/packages/37/57/ee347af605d867f712be7029bb94c8c071732a4b44792e3176fa3c612d39/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfb5862016acc9b869bb57284e6cb35fdf8e22fe59f7548858e2f971d045f150", size = 595506, upload-time = "2025-10-14T15:05:06.906Z" }, + { url = "https://files.pythonhosted.org/packages/a8/78/cc5ab0b86c122047f75e8fc471c67a04dee395daf847d3e59381996c8707/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:319b27255aacd9923b8a276bb14d21a5f7ff82564c744235fc5eae58d95422ae", size = 474936, upload-time = "2025-10-14T15:05:07.906Z" }, + { url = "https://files.pythonhosted.org/packages/62/da/def65b170a3815af7bd40a3e7010bf6ab53089ef1b75d05dd5385b87cf08/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c755367e51db90e75b19454b680903631d41f9e3607fbd941d296a020c2d752d", size = 456147, upload-time = "2025-10-14T15:05:09.138Z" }, + { url = "https://files.pythonhosted.org/packages/57/99/da6573ba71166e82d288d4df0839128004c67d2778d3b566c138695f5c0b/watchfiles-1.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c22c776292a23bfc7237a98f791b9ad3144b02116ff10d820829ce62dff46d0b", size = 630007, upload-time = "2025-10-14T15:05:10.117Z" }, + { url = "https://files.pythonhosted.org/packages/a8/51/7439c4dd39511368849eb1e53279cd3454b4a4dbace80bab88feeb83c6b5/watchfiles-1.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3a476189be23c3686bc2f4321dd501cb329c0a0469e77b7b534ee10129ae6374", size = 622280, upload-time = "2025-10-14T15:05:11.146Z" }, + { url = "https://files.pythonhosted.org/packages/95/9c/8ed97d4bba5db6fdcdb2b298d3898f2dd5c20f6b73aee04eabe56c59677e/watchfiles-1.1.1-cp313-cp313-win32.whl", hash = "sha256:bf0a91bfb5574a2f7fc223cf95eeea79abfefa404bf1ea5e339c0c1560ae99a0", size = 272056, upload-time = "2025-10-14T15:05:12.156Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f3/c14e28429f744a260d8ceae18bf58c1d5fa56b50d006a7a9f80e1882cb0d/watchfiles-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:52e06553899e11e8074503c8e716d574adeeb7e68913115c4b3653c53f9bae42", size = 288162, upload-time = "2025-10-14T15:05:13.208Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/fe0e56c40d5cd29523e398d31153218718c5786b5e636d9ae8ae79453d27/watchfiles-1.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:ac3cc5759570cd02662b15fbcd9d917f7ecd47efe0d6b40474eafd246f91ea18", size = 277909, upload-time = "2025-10-14T15:05:14.49Z" }, + { url = "https://files.pythonhosted.org/packages/79/42/e0a7d749626f1e28c7108a99fb9bf524b501bbbeb9b261ceecde644d5a07/watchfiles-1.1.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:563b116874a9a7ce6f96f87cd0b94f7faf92d08d0021e837796f0a14318ef8da", size = 403389, upload-time = "2025-10-14T15:05:15.777Z" }, + { url = "https://files.pythonhosted.org/packages/15/49/08732f90ce0fbbc13913f9f215c689cfc9ced345fb1bcd8829a50007cc8d/watchfiles-1.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ad9fe1dae4ab4212d8c91e80b832425e24f421703b5a42ef2e4a1e215aff051", size = 389964, upload-time = "2025-10-14T15:05:16.85Z" }, + { url = "https://files.pythonhosted.org/packages/27/0d/7c315d4bd5f2538910491a0393c56bf70d333d51bc5b34bee8e68e8cea19/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce70f96a46b894b36eba678f153f052967a0d06d5b5a19b336ab0dbbd029f73e", size = 448114, upload-time = "2025-10-14T15:05:17.876Z" }, + { url = "https://files.pythonhosted.org/packages/c3/24/9e096de47a4d11bc4df41e9d1e61776393eac4cb6eb11b3e23315b78b2cc/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb467c999c2eff23a6417e58d75e5828716f42ed8289fe6b77a7e5a91036ca70", size = 460264, upload-time = "2025-10-14T15:05:18.962Z" }, + { url = "https://files.pythonhosted.org/packages/cc/0f/e8dea6375f1d3ba5fcb0b3583e2b493e77379834c74fd5a22d66d85d6540/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:836398932192dae4146c8f6f737d74baeac8b70ce14831a239bdb1ca882fc261", size = 487877, upload-time = "2025-10-14T15:05:20.094Z" }, + { url = "https://files.pythonhosted.org/packages/ac/5b/df24cfc6424a12deb41503b64d42fbea6b8cb357ec62ca84a5a3476f654a/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:743185e7372b7bc7c389e1badcc606931a827112fbbd37f14c537320fca08620", size = 595176, upload-time = "2025-10-14T15:05:21.134Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b5/853b6757f7347de4e9b37e8cc3289283fb983cba1ab4d2d7144694871d9c/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afaeff7696e0ad9f02cbb8f56365ff4686ab205fcf9c4c5b6fdfaaa16549dd04", size = 473577, upload-time = "2025-10-14T15:05:22.306Z" }, + { url = "https://files.pythonhosted.org/packages/e1/f7/0a4467be0a56e80447c8529c9fce5b38eab4f513cb3d9bf82e7392a5696b/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f7eb7da0eb23aa2ba036d4f616d46906013a68caf61b7fdbe42fc8b25132e77", size = 455425, upload-time = "2025-10-14T15:05:23.348Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/82583485ea00137ddf69bc84a2db88bd92ab4a6e3c405e5fb878ead8d0e7/watchfiles-1.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:831a62658609f0e5c64178211c942ace999517f5770fe9436be4c2faeba0c0ef", size = 628826, upload-time = "2025-10-14T15:05:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/28/9a/a785356fccf9fae84c0cc90570f11702ae9571036fb25932f1242c82191c/watchfiles-1.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f9a2ae5c91cecc9edd47e041a930490c31c3afb1f5e6d71de3dc671bfaca02bf", size = 622208, upload-time = "2025-10-14T15:05:25.45Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f4/0872229324ef69b2c3edec35e84bd57a1289e7d3fe74588048ed8947a323/watchfiles-1.1.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:d1715143123baeeaeadec0528bb7441103979a1d5f6fd0e1f915383fea7ea6d5", size = 404315, upload-time = "2025-10-14T15:05:26.501Z" }, + { url = "https://files.pythonhosted.org/packages/7b/22/16d5331eaed1cb107b873f6ae1b69e9ced582fcf0c59a50cd84f403b1c32/watchfiles-1.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:39574d6370c4579d7f5d0ad940ce5b20db0e4117444e39b6d8f99db5676c52fd", size = 390869, upload-time = "2025-10-14T15:05:27.649Z" }, + { url = "https://files.pythonhosted.org/packages/b2/7e/5643bfff5acb6539b18483128fdc0ef2cccc94a5b8fbda130c823e8ed636/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7365b92c2e69ee952902e8f70f3ba6360d0d596d9299d55d7d386df84b6941fb", size = 449919, upload-time = "2025-10-14T15:05:28.701Z" }, + { url = "https://files.pythonhosted.org/packages/51/2e/c410993ba5025a9f9357c376f48976ef0e1b1aefb73b97a5ae01a5972755/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfff9740c69c0e4ed32416f013f3c45e2ae42ccedd1167ef2d805c000b6c71a5", size = 460845, upload-time = "2025-10-14T15:05:30.064Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a4/2df3b404469122e8680f0fcd06079317e48db58a2da2950fb45020947734/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b27cf2eb1dda37b2089e3907d8ea92922b673c0c427886d4edc6b94d8dfe5db3", size = 489027, upload-time = "2025-10-14T15:05:31.064Z" }, + { url = "https://files.pythonhosted.org/packages/ea/84/4587ba5b1f267167ee715b7f66e6382cca6938e0a4b870adad93e44747e6/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:526e86aced14a65a5b0ec50827c745597c782ff46b571dbfe46192ab9e0b3c33", size = 595615, upload-time = "2025-10-14T15:05:32.074Z" }, + { url = "https://files.pythonhosted.org/packages/6a/0f/c6988c91d06e93cd0bb3d4a808bcf32375ca1904609835c3031799e3ecae/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04e78dd0b6352db95507fd8cb46f39d185cf8c74e4cf1e4fbad1d3df96faf510", size = 474836, upload-time = "2025-10-14T15:05:33.209Z" }, + { url = "https://files.pythonhosted.org/packages/b4/36/ded8aebea91919485b7bbabbd14f5f359326cb5ec218cd67074d1e426d74/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c85794a4cfa094714fb9c08d4a218375b2b95b8ed1666e8677c349906246c05", size = 455099, upload-time = "2025-10-14T15:05:34.189Z" }, + { url = "https://files.pythonhosted.org/packages/98/e0/8c9bdba88af756a2fce230dd365fab2baf927ba42cd47521ee7498fd5211/watchfiles-1.1.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:74d5012b7630714b66be7b7b7a78855ef7ad58e8650c73afc4c076a1f480a8d6", size = 630626, upload-time = "2025-10-14T15:05:35.216Z" }, + { url = "https://files.pythonhosted.org/packages/2a/84/a95db05354bf2d19e438520d92a8ca475e578c647f78f53197f5a2f17aaf/watchfiles-1.1.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:8fbe85cb3201c7d380d3d0b90e63d520f15d6afe217165d7f98c9c649654db81", size = 622519, upload-time = "2025-10-14T15:05:36.259Z" }, + { url = "https://files.pythonhosted.org/packages/1d/ce/d8acdc8de545de995c339be67711e474c77d643555a9bb74a9334252bd55/watchfiles-1.1.1-cp314-cp314-win32.whl", hash = "sha256:3fa0b59c92278b5a7800d3ee7733da9d096d4aabcfabb9a928918bd276ef9b9b", size = 272078, upload-time = "2025-10-14T15:05:37.63Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c9/a74487f72d0451524be827e8edec251da0cc1fcf111646a511ae752e1a3d/watchfiles-1.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:c2047d0b6cea13b3316bdbafbfa0c4228ae593d995030fda39089d36e64fc03a", size = 287664, upload-time = "2025-10-14T15:05:38.95Z" }, + { url = "https://files.pythonhosted.org/packages/df/b8/8ac000702cdd496cdce998c6f4ee0ca1f15977bba51bdf07d872ebdfc34c/watchfiles-1.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:842178b126593addc05acf6fce960d28bc5fae7afbaa2c6c1b3a7b9460e5be02", size = 277154, upload-time = "2025-10-14T15:05:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/47/a8/e3af2184707c29f0f14b1963c0aace6529f9d1b8582d5b99f31bbf42f59e/watchfiles-1.1.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:88863fbbc1a7312972f1c511f202eb30866370ebb8493aef2812b9ff28156a21", size = 403820, upload-time = "2025-10-14T15:05:40.932Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/e47e307c2f4bd75f9f9e8afbe3876679b18e1bcec449beca132a1c5ffb2d/watchfiles-1.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:55c7475190662e202c08c6c0f4d9e345a29367438cf8e8037f3155e10a88d5a5", size = 390510, upload-time = "2025-10-14T15:05:41.945Z" }, + { url = "https://files.pythonhosted.org/packages/d5/a0/ad235642118090f66e7b2f18fd5c42082418404a79205cdfca50b6309c13/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f53fa183d53a1d7a8852277c92b967ae99c2d4dcee2bfacff8868e6e30b15f7", size = 448408, upload-time = "2025-10-14T15:05:43.385Z" }, + { url = "https://files.pythonhosted.org/packages/df/85/97fa10fd5ff3332ae17e7e40e20784e419e28521549780869f1413742e9d/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6aae418a8b323732fa89721d86f39ec8f092fc2af67f4217a2b07fd3e93c6101", size = 458968, upload-time = "2025-10-14T15:05:44.404Z" }, + { url = "https://files.pythonhosted.org/packages/47/c2/9059c2e8966ea5ce678166617a7f75ecba6164375f3b288e50a40dc6d489/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f096076119da54a6080e8920cbdaac3dbee667eb91dcc5e5b78840b87415bd44", size = 488096, upload-time = "2025-10-14T15:05:45.398Z" }, + { url = "https://files.pythonhosted.org/packages/94/44/d90a9ec8ac309bc26db808a13e7bfc0e4e78b6fc051078a554e132e80160/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00485f441d183717038ed2e887a7c868154f216877653121068107b227a2f64c", size = 596040, upload-time = "2025-10-14T15:05:46.502Z" }, + { url = "https://files.pythonhosted.org/packages/95/68/4e3479b20ca305cfc561db3ed207a8a1c745ee32bf24f2026a129d0ddb6e/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a55f3e9e493158d7bfdb60a1165035f1cf7d320914e7b7ea83fe22c6023b58fc", size = 473847, upload-time = "2025-10-14T15:05:47.484Z" }, + { url = "https://files.pythonhosted.org/packages/4f/55/2af26693fd15165c4ff7857e38330e1b61ab8c37d15dc79118cdba115b7a/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c91ed27800188c2ae96d16e3149f199d62f86c7af5f5f4d2c61a3ed8cd3666c", size = 455072, upload-time = "2025-10-14T15:05:48.928Z" }, + { url = "https://files.pythonhosted.org/packages/66/1d/d0d200b10c9311ec25d2273f8aad8c3ef7cc7ea11808022501811208a750/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:311ff15a0bae3714ffb603e6ba6dbfba4065ab60865d15a6ec544133bdb21099", size = 629104, upload-time = "2025-10-14T15:05:49.908Z" }, + { url = "https://files.pythonhosted.org/packages/e3/bd/fa9bb053192491b3867ba07d2343d9f2252e00811567d30ae8d0f78136fe/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a916a2932da8f8ab582f242c065f5c81bed3462849ca79ee357dd9551b0e9b01", size = 622112, upload-time = "2025-10-14T15:05:50.941Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8e/e500f8b0b77be4ff753ac94dc06b33d8f0d839377fee1b78e8c8d8f031bf/watchfiles-1.1.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:db476ab59b6765134de1d4fe96a1a9c96ddf091683599be0f26147ea1b2e4b88", size = 408250, upload-time = "2025-10-14T15:06:10.264Z" }, + { url = "https://files.pythonhosted.org/packages/bd/95/615e72cd27b85b61eec764a5ca51bd94d40b5adea5ff47567d9ebc4d275a/watchfiles-1.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:89eef07eee5e9d1fda06e38822ad167a044153457e6fd997f8a858ab7564a336", size = 396117, upload-time = "2025-10-14T15:06:11.28Z" }, + { url = "https://files.pythonhosted.org/packages/c9/81/e7fe958ce8a7fb5c73cc9fb07f5aeaf755e6aa72498c57d760af760c91f8/watchfiles-1.1.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce19e06cbda693e9e7686358af9cd6f5d61312ab8b00488bc36f5aabbaf77e24", size = 450493, upload-time = "2025-10-14T15:06:12.321Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d4/ed38dd3b1767193de971e694aa544356e63353c33a85d948166b5ff58b9e/watchfiles-1.1.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e6f39af2eab0118338902798b5aa6664f46ff66bc0280de76fca67a7f262a49", size = 457546, upload-time = "2025-10-14T15:06:13.372Z" }, +] + [[package]] name = "websockets" version = "15.0.1"