From 2b49526d073faa7d70484d79206c2df85a66c93e Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Fri, 19 Dec 2025 14:44:06 +0100 Subject: [PATCH 001/129] Simplified way of implementing fields --- src/verification/__init__.py | 6 +++++- workflow/scripts/verif_single_init.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index 5f7c7a00..f0df171f 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -203,7 +203,11 @@ def verify( scores = _merge_metrics(scores) statistics = _merge_metrics(statistics) - out = xr.merge([scores, statistics], join="outer", compat="no_conflicts") + out = xr.merge( + [scores, statistics, fcst_aligned - obs_aligned], + join="outer", + compat="no_conflicts", + ) LOG.info("Computed metrics in %.2f seconds", time.time() - start) LOG.info("Metrics dataset: \n%s", out) return out diff --git a/workflow/scripts/verif_single_init.py b/workflow/scripts/verif_single_init.py index 57b1e2c4..52d9e08f 100644 --- a/workflow/scripts/verif_single_init.py +++ b/workflow/scripts/verif_single_init.py @@ -114,6 +114,7 @@ def main(args: ScriptConfig): # compute metrics and statistics results = verify(fcst, analysis, args.label, args.analysis_label, args.regions) + LOG.info("Verification results:\n%s", results) # save results to NetCDF args.output.parent.mkdir(parents=True, exist_ok=True) From a456b1d5e3806f58178f83c74f9263a05589ca9e Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Fri, 19 Dec 2025 15:56:16 +0100 Subject: [PATCH 002/129] Exclude spatial data from being plotted and included in dashboard --- workflow/scripts/report_experiment_dashboard.py | 2 +- workflow/scripts/verif_plot_metrics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/scripts/report_experiment_dashboard.py b/workflow/scripts/report_experiment_dashboard.py index 327547b0..a2abe766 100644 --- a/workflow/scripts/report_experiment_dashboard.py +++ b/workflow/scripts/report_experiment_dashboard.py @@ -39,7 +39,7 @@ def main(args): LOG.info("Loaded verification netcdf: \n%s", ds) # extract only non-spatial variables to pd.DataFrame - nonspatial_vars = [d for d in ds.data_vars if "spatial" not in d] + nonspatial_vars = [d for d in ds.data_vars if "x" not in ds[d].dims] df = ds[nonspatial_vars].to_array("stack").to_dataframe(name="value").reset_index() df[["param", "metric"]] = df["stack"].str.split(".", n=1, expand=True) df.drop(columns=["stack"], inplace=True) diff --git a/workflow/scripts/verif_plot_metrics.py b/workflow/scripts/verif_plot_metrics.py index a5a50bba..ae6d9d66 100644 --- a/workflow/scripts/verif_plot_metrics.py +++ b/workflow/scripts/verif_plot_metrics.py @@ -84,7 +84,7 @@ def main(args: Namespace) -> None: ds = xr.concat(dfs, dim="source", join="outer") # extract only non-spatial variables to pd.DataFrame - nonspatial_vars = [d for d in ds.data_vars if "spatial" not in d] + nonspatial_vars = [d for d in ds.data_vars if "x" not in ds[d].dims] all_df = ( ds[nonspatial_vars].to_array("stack").to_dataframe(name="value").reset_index() ) From 0ec286f4b24fe81f226bbd5ac9c56273ec6bceb8 Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Wed, 7 Jan 2026 11:28:55 +0100 Subject: [PATCH 003/129] delete intermeidate verification files --- workflow/rules/verif.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index cba4a307..872f43eb 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -27,7 +27,7 @@ rule verif_metrics_baseline: analysis_label=config["analysis"].get("label"), regions=REGION_TXT, output: - OUT_ROOT / "data/baselines/{baseline_id}/{init_time}/verif.nc", + temp(OUT_ROOT / "data/baselines/{baseline_id}/{init_time}/verif.nc"), log: OUT_ROOT / "logs/verif_metrics_baseline/{baseline_id}-{init_time}.log", resources: @@ -76,7 +76,7 @@ rule verif_metrics: Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" ).resolve(), log: - OUT_ROOT / "logs/verif_metrics/{run_id}-{init_time}.log", + temp(OUT_ROOT / "logs/verif_metrics/{run_id}-{init_time}.log"), resources: cpus_per_task=24, mem_mb=50_000, From 28692d6c6adf890ab88554f6832d9ba9ded457c0 Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Wed, 7 Jan 2026 12:02:58 +0100 Subject: [PATCH 004/129] Fix typo --- workflow/rules/verif.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index 872f43eb..0566ccf2 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -63,7 +63,7 @@ rule verif_metrics: inference_okfile=rules.execute_inference.output.okfile, analysis_zarr=config["analysis"].get("analysis_zarr"), output: - OUT_ROOT / "data/runs/{run_id}/{init_time}/verif.nc", + temp(OUT_ROOT / "data/runs/{run_id}/{init_time}/verif.nc"), # wildcard_constraints: # run_id="^" # to avoid ambiguitiy with run_baseline_verif # TODO: implement logic to use experiment name instead of run_id as wildcard @@ -76,7 +76,7 @@ rule verif_metrics: Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" ).resolve(), log: - temp(OUT_ROOT / "logs/verif_metrics/{run_id}-{init_time}.log"), + OUT_ROOT / "logs/verif_metrics/{run_id}-{init_time}.log", resources: cpus_per_task=24, mem_mb=50_000, From f3dcf0db6b5b53f6a2190590182a380deca0ab14 Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Wed, 7 Jan 2026 12:04:04 +0100 Subject: [PATCH 005/129] include score components for maps --- src/verification/__init__.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index f0df171f..51e483a4 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -198,13 +198,24 @@ def verify( score = xr.concat(score, dim="region") fcst_statistics = xr.concat(fcst_statistics, dim="region") obs_statistics = xr.concat(obs_statistics, dim="region") + score_spatial = _compute_scores( + fcst_aligned[param], + obs_aligned[param], + prefix=param + ".", + suffix=".spatial", + dim=[], + ) statistics.append(xr.concat([fcst_statistics, obs_statistics], dim="source")) - scores.append(score) + scores.append( + xr.merge([score, score_spatial], join="outer", compat="no_conflicts") + ) scores = _merge_metrics(scores) statistics = _merge_metrics(statistics) + LOG.info("Computed scores dataset: \n%s", scores) + LOG.info("Computed statistics dataset: \n%s", statistics) out = xr.merge( - [scores, statistics, fcst_aligned - obs_aligned], + [scores, statistics], join="outer", compat="no_conflicts", ) From 99dac523cdb71355c670088d4d15f72a1f8074bb Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Wed, 7 Jan 2026 17:09:53 +0100 Subject: [PATCH 006/129] Revert "Exclude spatial data from being plotted and included in dashboard" This reverts commit cdefa16a7f0cc8d9721b7598e49a88800ab61d23. --- workflow/scripts/report_experiment_dashboard.py | 2 +- workflow/scripts/verif_plot_metrics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/scripts/report_experiment_dashboard.py b/workflow/scripts/report_experiment_dashboard.py index a2abe766..327547b0 100644 --- a/workflow/scripts/report_experiment_dashboard.py +++ b/workflow/scripts/report_experiment_dashboard.py @@ -39,7 +39,7 @@ def main(args): LOG.info("Loaded verification netcdf: \n%s", ds) # extract only non-spatial variables to pd.DataFrame - nonspatial_vars = [d for d in ds.data_vars if "x" not in ds[d].dims] + nonspatial_vars = [d for d in ds.data_vars if "spatial" not in d] df = ds[nonspatial_vars].to_array("stack").to_dataframe(name="value").reset_index() df[["param", "metric"]] = df["stack"].str.split(".", n=1, expand=True) df.drop(columns=["stack"], inplace=True) diff --git a/workflow/scripts/verif_plot_metrics.py b/workflow/scripts/verif_plot_metrics.py index ae6d9d66..a5a50bba 100644 --- a/workflow/scripts/verif_plot_metrics.py +++ b/workflow/scripts/verif_plot_metrics.py @@ -84,7 +84,7 @@ def main(args: Namespace) -> None: ds = xr.concat(dfs, dim="source", join="outer") # extract only non-spatial variables to pd.DataFrame - nonspatial_vars = [d for d in ds.data_vars if "x" not in ds[d].dims] + nonspatial_vars = [d for d in ds.data_vars if "spatial" not in d] all_df = ( ds[nonspatial_vars].to_array("stack").to_dataframe(name="value").reset_index() ) From edcca5b6caff4cd470700c170ee3fe661e77fcaf Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Wed, 7 Jan 2026 20:17:29 +0100 Subject: [PATCH 007/129] remove source dimension from scores --- src/verification/__init__.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index 51e483a4..e9951b8a 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -84,17 +84,26 @@ def _compute_scores( """ dim = ["x", "y"] if "x" in fcst.dims and "y" in fcst.dims else ["values"] error = fcst - obs - scores = xr.Dataset( - { - f"{prefix}BIAS{suffix}": error.mean(dim=dim, skipna=True), - f"{prefix}MSE{suffix}": (error**2).mean(dim=dim, skipna=True), - f"{prefix}MAE{suffix}": abs(error).mean(dim=dim, skipna=True), - f"{prefix}VAR{suffix}": error.var(dim=dim, skipna=True), - f"{prefix}CORR{suffix}": xr.corr(fcst, obs, dim=dim), - f"{prefix}R2{suffix}": xr.corr(fcst, obs, dim=dim) ** 2, - } - ) - scores = scores.expand_dims({"source": [source]}) + if dim == []: + scores = xr.Dataset( + { + f"{prefix}BIAS{suffix}": error, + f"{prefix}MSE{suffix}": (error**2), + f"{prefix}MAE{suffix}": abs(error), + } + ) + else: + scores = xr.Dataset( + { + f"{prefix}BIAS{suffix}": error.mean(dim=dim, skipna=True), + f"{prefix}MSE{suffix}": (error**2).mean(dim=dim, skipna=True), + f"{prefix}MAE{suffix}": abs(error).mean(dim=dim, skipna=True), + f"{prefix}VAR{suffix}": error.var(dim=dim, skipna=True), + f"{prefix}CORR{suffix}": xr.corr(fcst, obs, dim=dim), + f"{prefix}R2{suffix}": xr.corr(fcst, obs, dim=dim) ** 2, + } + ) + # scores = scores.expand_dims({"source": [source]}) return scores @@ -212,8 +221,6 @@ def verify( scores = _merge_metrics(scores) statistics = _merge_metrics(statistics) - LOG.info("Computed scores dataset: \n%s", scores) - LOG.info("Computed statistics dataset: \n%s", statistics) out = xr.merge( [scores, statistics], join="outer", From 0ec8a0f926bd4b641c1436aa4793e1c8592da515 Mon Sep 17 00:00:00 2001 From: Jonas Bhend Date: Thu, 8 Jan 2026 08:40:39 +0100 Subject: [PATCH 008/129] clean up --- src/verification/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index e9951b8a..8f05349c 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -221,11 +221,7 @@ def verify( scores = _merge_metrics(scores) statistics = _merge_metrics(statistics) - out = xr.merge( - [scores, statistics], - join="outer", - compat="no_conflicts", - ) + out = xr.merge([scores, statistics], join="outer", compat="no_conflicts") LOG.info("Computed metrics in %.2f seconds", time.time() - start) LOG.info("Metrics dataset: \n%s", out) return out From 51f22a6e44cf5f2660a7813cba1bbf73e7ab4614 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 12 Jan 2026 16:28:01 +0100 Subject: [PATCH 009/129] New rule and plotting file for plotting maps of summary statistics. (No changes to code yet.) --- workflow/rules/plot.smk | 32 +++ workflow/scripts/plot_summary_stat_maps.mo.py | 239 ++++++++++++++++++ 2 files changed, 271 insertions(+) create mode 100644 workflow/scripts/plot_summary_stat_maps.mo.py diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index d4def1ba..7fbd3320 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -67,3 +67,35 @@ rule make_forecast_animation: """ convert -delay {params.delay} -loop 0 {input} {output} """ + + +rule plot_summary_stat_maps: + input: + script="workflow/scripts/plot_forecast_frame.mo.py", + inference_okfile=rules.execute_inference.output.okfile, + output: + temp( + OUT_ROOT + / "showcases/{run_id}/{init_time}/frames/{init_time}_{leadtime}_{param}_{region}.png" + ), + wildcard_constraints: + leadtime=r"\d+", # only digits + resources: + slurm_partition="postproc", + cpus_per_task=1, + runtime="10m", + params: + grib_out_dir=lambda wc: ( + Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" + ).resolve(), + shell: + """ + export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) + python {input.script} \ + --input {params.grib_out_dir} --date {wildcards.init_time} --outfn {output[0]} \ + --param {wildcards.param} --leadtime {wildcards.leadtime} --region {wildcards.region} \ + # interactive editing (needs to set localrule: True and use only one core) + # marimo edit {input.script} -- \ + # --input {params.grib_out_dir} --date {wildcards.init_time} --outfn {output[0]}\ + # --param {wildcards.param} --leadtime {wildcards.leadtime} --region {wildcards.region}\ + """ \ No newline at end of file diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py new file mode 100644 index 00000000..b8592141 --- /dev/null +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -0,0 +1,239 @@ +import marimo + +__generated_with = "0.16.5" +app = marimo.App(width="medium") + + +@app.cell +def _(): + import logging + from argparse import ArgumentParser + from pathlib import Path + + import cartopy.crs as ccrs + import earthkit.plots as ekp + import numpy as np + + from plotting import DOMAINS + from plotting import StatePlotter + from plotting.colormap_defaults import CMAP_DEFAULTS + from plotting.compat import load_state_from_grib + + return ( + ArgumentParser, + CMAP_DEFAULTS, + Path, + StatePlotter, + ekp, + load_state_from_grib, + logging, + np, + DOMAINS, + ccrs, + ) + + +@app.cell +def _(logging): + LOG = logging.getLogger(__name__) + LOG_FMT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + logging.basicConfig(level=logging.INFO, format=LOG_FMT) + return (LOG,) + + +@app.cell +def _(ArgumentParser, Path): + parser = ArgumentParser() + + parser.add_argument( + "--input", type=str, default=None, help="Directory to grib data" + ) + parser.add_argument("--date", type=str, default=None, help="reference datetime") + parser.add_argument("--outfn", type=str, help="output filename") + parser.add_argument("--leadtime", type=str, help="leadtime") + parser.add_argument("--param", type=str, help="parameter") + parser.add_argument("--region", type=str, help="name of region") + + args = parser.parse_args() + grib_dir = Path(args.input) + init_time = args.date + outfn = Path(args.outfn) + lead_time = args.leadtime + param = args.param + region = args.region + return ( + args, + grib_dir, + init_time, + lead_time, + outfn, + param, + region, + ) + + +@app.cell +def _(grib_dir, init_time, lead_time, load_state_from_grib, param): + # load grib file + grib_file = grib_dir / f"{init_time}_{lead_time}.grib" + if param == "SP_10M": + paramlist = ["U_10M", "V_10M"] + elif param == "SP": + paramlist = ["U", "V"] + else: + paramlist = [param] + state = load_state_from_grib(grib_file, paramlist=paramlist) + return (state,) + + +@app.cell +def _(CMAP_DEFAULTS, ekp): + def get_style(param, units_override=None): + """Get style and colormap settings for the plot. + Needed because cmap/norm does not work in Style(colors=cmap), + still needs to be passed as arguments to tripcolor()/tricontourf(). + """ + cfg = CMAP_DEFAULTS[param] + units = units_override if units_override is not None else cfg.get("units", "") + return { + "style": ekp.styles.Style( + levels=cfg.get("bounds", cfg.get("levels", None)), + extend="both", + units=units, + colors=cfg.get("colors", None), + ), + "norm": cfg.get("norm", None), + "cmap": cfg.get("cmap", None), + "levels": cfg.get("levels", None), + "vmin": cfg.get("vmin", None), + "vmax": cfg.get("vmax", None), + "colors": cfg.get("colors", None), + } + + return (get_style,) + + +@app.cell +def _(LOG, np): + """Preprocess fields with pint-based unit conversion and derived quantities.""" + try: + import pint # type: ignore + + _ureg = pint.UnitRegistry() + + def _k_to_c(arr): + # robust conversion with pint, fallback if dtype unsupported + try: + return (_ureg.Quantity(arr, _ureg.kelvin).to(_ureg.degC)).magnitude + except Exception: + return arr - 273.15 + + def _ms_to_knots(arr): + # robust conversion with pint, fallback if dtype unsupported + try: + return ( + _ureg.Quantity(arr, _ureg.meter / _ureg.second).to(_ureg.knot) + ).magnitude + except Exception: + return arr * 1.943844 + + def _m_to_mm(arr): + # robust conversion with pint, fallback if dtype unsupported + try: + return (_ureg.Quantity(arr, _ureg.meter).to(_ureg.millimeter)).magnitude + except Exception: + return arr * 1000 + + except Exception: + LOG.warning("pint not available; falling back hardcoded conversions") + + def _k_to_c(arr): + return arr - 273.15 + + def _ms_to_knots(arr): + return arr * 1.943844 + + def _m_to_mm(arr): + return arr * 1000 + + def preprocess_field(param: str, state: dict): + """ + - Temperatures: K -> °C + - Wind speed: sqrt(u^2 + v^2) + - Precipitation: m -> mm + Returns: (field_array, units_override or None) + """ + fields = state["fields"] + # temperature variables + if param in ("T_2M", "TD_2M", "T", "TD"): + return _k_to_c(fields[param]), "°C" + # 10m wind speed (allow legacy 'uv' alias) + if param == "SP_10M": + u = fields["U_10M"] + v = fields["V_10M"] + return np.sqrt(u**2 + v**2), "m/s" + # wind speed from standard-level components + if param == "SP": + u = fields["U"] + v = fields["V"] + return np.sqrt(u**2 + v**2), "m/s" + if param == "TOT_PREC": + return _m_to_mm(fields[param]), "mm" + # default: passthrough + return fields[param], None + + return (preprocess_field,) + + +@app.cell +def _( + LOG, + StatePlotter, + args, + get_style, + outfn, + param, + preprocess_field, + region, + state, + DOMAINS, + ccrs, +): + # plot individual fields + plotter = StatePlotter( + state["longitudes"], + state["latitudes"], + outfn.parent, + ) + fig = plotter.init_geoaxes( + nrows=1, + ncols=1, + projection=DOMAINS[region]["projection"], + bbox=DOMAINS[region]["extent"], + name=region, + size=(6, 6), + ) + subplot = fig.add_map(row=0, column=0) + + # preprocess field (unit conversion, derived quantities) + field, units_override = preprocess_field(param, state) + + plotter.plot_field(subplot, field, **get_style(args.param, units_override)) + subplot.ax.add_geometries( + state["lam_envelope"], + edgecolor="black", + facecolor="none", + crs=ccrs.PlateCarree(), + ) + validtime = state["valid_time"].strftime("%Y%m%d%H%M") + # leadtime = int(state["lead_time"].total_seconds() // 3600) + + fig.title(f"{param}, time: {validtime}") + + fig.save(outfn, bbox_inches="tight", dpi=200) + LOG.info(f"saved: {outfn}") + return + + +if __name__ == "__main__": + app.run() From 366249d5563caa9dc1d795d25ee84f65b35fe952 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 12 Jan 2026 17:01:42 +0100 Subject: [PATCH 010/129] Obvious changes to the new plotting rule for maps of summary statistics. --- workflow/rules/plot.smk | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 7fbd3320..fee1dfe3 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -71,13 +71,10 @@ rule make_forecast_animation: rule plot_summary_stat_maps: input: - script="workflow/scripts/plot_forecast_frame.mo.py", + script="workflow/scripts/plot_summary_stat_maps.mo.py", inference_okfile=rules.execute_inference.output.okfile, output: - temp( - OUT_ROOT - / "showcases/{run_id}/{init_time}/frames/{init_time}_{leadtime}_{param}_{region}.png" - ), + OUT_ROOT / "results/summary_stats/maps/{run_id}/{leadtime}/{metric}_{param}_{region}.png", wildcard_constraints: leadtime=r"\d+", # only digits resources: @@ -85,9 +82,7 @@ rule plot_summary_stat_maps: cpus_per_task=1, runtime="10m", params: - grib_out_dir=lambda wc: ( - Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" - ).resolve(), + # What do I need here? shell: """ export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) From f53139519f83168321f6db96f19e228772958aec Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 12 Jan 2026 17:30:14 +0100 Subject: [PATCH 011/129] Some more changes (preliminary, to be continued). --- workflow/rules/plot.smk | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index fee1dfe3..8090a213 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -82,12 +82,14 @@ rule plot_summary_stat_maps: cpus_per_task=1, runtime="10m", params: - # What do I need here? + nc_out_dir=lambda wc: ( + Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" # to be adjusted. + ).resolve(), shell: """ export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) python {input.script} \ - --input {params.grib_out_dir} --date {wildcards.init_time} --outfn {output[0]} \ + --input {params.nc_out_dir} --date {wildcards.init_time} --outfn {output[0]} \ --param {wildcards.param} --leadtime {wildcards.leadtime} --region {wildcards.region} \ # interactive editing (needs to set localrule: True and use only one core) # marimo edit {input.script} -- \ From 32c7ecf6ff676c9c978090f44208bfd1e53ab558 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 13 Jan 2026 18:02:48 +0100 Subject: [PATCH 012/129] Further changes to plotting scripts. --- workflow/rules/plot.smk | 4 +++- workflow/scripts/plot_summary_stat_maps.mo.py | 14 +++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 8090a213..2014e000 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -83,7 +83,9 @@ rule plot_summary_stat_maps: runtime="10m", params: nc_out_dir=lambda wc: ( - Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" # to be adjusted. + Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" + # not sure how to do this, because the baselines are in, e.g., output/data/baselines/COSMO-E/verif_aggregated.nc + # and the runs are in output/data/runs/runID/verif_aggregated.nc ).resolve(), shell: """ diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index b8592141..4b8eb38f 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -6,18 +6,30 @@ @app.cell def _(): + + # this sure stays the same. import logging from argparse import ArgumentParser from pathlib import Path + # this sure stays the same. import cartopy.crs as ccrs import earthkit.plots as ekp import numpy as np + # this stays the same as well. from plotting import DOMAINS + + # no changes to StatePlotter required according to ChatGPT. from plotting import StatePlotter + + # Probably need some new colour maps. + # at least one for biases (diverging), maybe different diverging ones for the different variables. + # from plotting.colormap_defaults import CMAP_DEFAULTS - from plotting.compat import load_state_from_grib + + # need to load nc files... + from plotting.compat import load_state_from_grib # TODO: load state from nc? return ( ArgumentParser, From c2ab6451d822271cdd4a9598c3dec91ae118134c Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 14 Jan 2026 15:12:43 +0100 Subject: [PATCH 013/129] First version of colour maps finished. For Bias, RMSE and MAE map plots. --- src/plotting/colormap_defaults.py | 45 ++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 52ff0502..4a85f67e 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -4,12 +4,23 @@ from matplotlib import pyplot as plt import warnings from .colormap_loader import load_ncl_colormap - +from matplotlib.colors import BoundaryNorm +import numpy as np def _fallback(): warnings.warn("No colormap found for this parameter, using fallback.", UserWarning) return {"cmap": plt.get_cmap("viridis"), "norm": None, "units": ""} +def symmetric_boundary_norm(nlevels): + """ + Returns a callable that creates a symmetric BoundaryNorm + around zero with `nlevels` discrete colors. Used for creating colormaps for bias. + """ + def _norm(data): + vmax = np.nanmax(np.abs(data)) + boundaries = np.linspace(-vmax, vmax, nlevels + 1) + return BoundaryNorm(boundaries=boundaries, ncolors=nlevels) + return _norm _CMAP_DEFAULTS = { "SP": {"cmap": plt.get_cmap("coolwarm", 11), "vmin": 800 * 100, "vmax": 1100 * 100}, @@ -44,6 +55,38 @@ def _fallback(): "units": "mm", "levels": [0, 0.05, 0.1, 0.25, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 100], }, + + # hard-code this for the moment, can still make smarter later on: + # RMSE and MAE first (is all the same). Use Reds colormap to indicate 'error'. + # Use a limited number of levels so that absolute values of error can be read from the map. + # always start at 0 so that the saturation of the colour corresponds to the error magnitude. + + # RMSE: + "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "m/s"}, + "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "m/s"}, + "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "°C"}, + "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "°C"}, + "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "Pa"}, + "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "Pa"}, + "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "mm"}, + + # MAE: + "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "m/s"}, + "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "m/s"}, + "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "°C"}, + "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "°C"}, + "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "Pa"}, + "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "Pa"}, + "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "mm"}, + + # Bias: + "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "m/s"}, + "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "m/s"}, + "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "°C"}, + "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "°C"}, + "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "Pa"}, + "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "Pa"}, + "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "mm"} } CMAP_DEFAULTS = defaultdict(_fallback, _CMAP_DEFAULTS) From 27b91e2d123af3a1b142f73e9cf65266db8ec91e Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 14 Jan 2026 17:55:36 +0100 Subject: [PATCH 014/129] Better comments in the colour map code. --- src/plotting/colormap_defaults.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 4a85f67e..5e22f72f 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -57,7 +57,8 @@ def _norm(data): }, # hard-code this for the moment, can still make smarter later on: - # RMSE and MAE first (is all the same). Use Reds colormap to indicate 'error'. + # RMSE and MAE first (is all the same). Sequential colour map to reflect the nature of the data (error, all positive). + # Red is suggestive of 'bad' (high error). # Use a limited number of levels so that absolute values of error can be read from the map. # always start at 0 so that the saturation of the colour corresponds to the error magnitude. @@ -80,6 +81,8 @@ def _norm(data): "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "mm"}, # Bias: + # diverging colour scheme for the Bias to reflect the nature of the data (can be positive or negative, symmetric). + # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "m/s"}, "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "°C"}, From 1b2e6702e4bdf4f976da32d5b96981e372073f65 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 14 Jan 2026 17:56:39 +0100 Subject: [PATCH 015/129] Better Comments, some further changes to code. --- workflow/scripts/plot_summary_stat_maps.mo.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 4b8eb38f..14eb182a 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -23,13 +23,12 @@ def _(): # no changes to StatePlotter required according to ChatGPT. from plotting import StatePlotter - # Probably need some new colour maps. - # at least one for biases (diverging), maybe different diverging ones for the different variables. - # + # Added some new colour maps for the Bias / MAE / RMSE map plots. from plotting.colormap_defaults import CMAP_DEFAULTS - # need to load nc files... - from plotting.compat import load_state_from_grib # TODO: load state from nc? + # need to load nc files. But this statement is not needed any more because + # the .nc files can just be read with xr.open_dataset + # from plotting.compat import load_state_from_grib return ( ArgumentParser, @@ -37,7 +36,7 @@ def _(): Path, StatePlotter, ekp, - load_state_from_grib, + # load_state_from_grib, logging, np, DOMAINS, @@ -58,25 +57,23 @@ def _(ArgumentParser, Path): parser = ArgumentParser() parser.add_argument( - "--input", type=str, default=None, help="Directory to grib data" + "--input", type=str, default=None, help="Directory to .nc data containing the error fields" ) - parser.add_argument("--date", type=str, default=None, help="reference datetime") + parser.add_argument("--outfn", type=str, help="output filename") parser.add_argument("--leadtime", type=str, help="leadtime") parser.add_argument("--param", type=str, help="parameter") parser.add_argument("--region", type=str, help="name of region") args = parser.parse_args() - grib_dir = Path(args.input) - init_time = args.date + nc_dir = Path(args.input) outfn = Path(args.outfn) lead_time = args.leadtime param = args.param region = args.region return ( args, - grib_dir, - init_time, + nc_dir, lead_time, outfn, param, @@ -85,9 +82,9 @@ def _(ArgumentParser, Path): @app.cell -def _(grib_dir, init_time, lead_time, load_state_from_grib, param): +def _(nc_dir, init_time, lead_time, load_state_from_grib, param): # load grib file - grib_file = grib_dir / f"{init_time}_{lead_time}.grib" + grib_file = nc_dir / f"{init_time}_{lead_time}.grib" if param == "SP_10M": paramlist = ["U_10M", "V_10M"] elif param == "SP": From 52dd1e71e7a84f00e8197e2b68a78176270691c3 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 15 Jan 2026 15:46:47 +0100 Subject: [PATCH 016/129] Added back instances of lead time. --- workflow/scripts/plot_summary_stat_maps.mo.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 14eb182a..2a60de34 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -59,7 +59,7 @@ def _(ArgumentParser, Path): parser.add_argument( "--input", type=str, default=None, help="Directory to .nc data containing the error fields" ) - + # parser.add_argument("--date", type=str, default=None, help="reference datetime") # to be deleted? parser.add_argument("--outfn", type=str, help="output filename") parser.add_argument("--leadtime", type=str, help="leadtime") parser.add_argument("--param", type=str, help="parameter") @@ -67,6 +67,7 @@ def _(ArgumentParser, Path): args = parser.parse_args() nc_dir = Path(args.input) + # init_time = args.date # to be deleted? outfn = Path(args.outfn) lead_time = args.leadtime param = args.param @@ -74,6 +75,7 @@ def _(ArgumentParser, Path): return ( args, nc_dir, + # init_time, # to be deleted? lead_time, outfn, param, @@ -83,8 +85,8 @@ def _(ArgumentParser, Path): @app.cell def _(nc_dir, init_time, lead_time, load_state_from_grib, param): - # load grib file - grib_file = nc_dir / f"{init_time}_{lead_time}.grib" + # load .nc verification file + nc_file = nc_dir / "verif_aggregated.nc" if param == "SP_10M": paramlist = ["U_10M", "V_10M"] elif param == "SP": From ddc5883ce3e953af50a93ef10eb7d53f3c349e3e Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 15 Jan 2026 17:55:59 +0100 Subject: [PATCH 017/129] New function for loading netCDF files added to compat.py --- src/plotting/compat.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/src/plotting/compat.py b/src/plotting/compat.py index 665287e0..fbe26f3e 100644 --- a/src/plotting/compat.py +++ b/src/plotting/compat.py @@ -90,3 +90,79 @@ def load_state_from_raw( if key.startswith("field_"): state["fields"][key.removeprefix("field_")] = value return state + + +def load_state_from_netcdf( + file: Path, + paramlist: list[str], + *, + season: str = "all", + init_hour: int = -999, + lead_time: int | None = None, # hours +) -> dict: + """ + NetCDF analogue of load_state_from_grib(), restricted to spatial variables. + """ + + ds = xr.open_dataset(file) + + # --- normalize lead_time to hours (float) --- + if ds["lead_time"].dtype.kind == "m": + ds = ds.assign_coords( + lead_time=ds["lead_time"].dt.total_seconds() / 3600 + ) + + # --- select season / init_hour / lead_time --- + ds = ds.sel(season=season, init_hour=init_hour) + if lead_time is not None: + ds = ds.sel(lead_time=lead_time) + + # --- infer reference + valid time --- + # Assumption: forecast_reference_time is not explicitly stored + # We reconstruct something consistent with GRIB usage + forecast_reference_time = None + valid_time = None + if lead_time is not None: + valid_time = pd.to_datetime(lead_time, unit="h", origin="unix") + + # --- get lat / lon (assumed present as coordinates) --- + lat = ds["lat"].values if "lat" in ds.coords else ds["latitude"].values + lon = ds["lon"].values if "lon" in ds.coords else ds["longitude"].values + + lon2d, lat2d = np.meshgrid(lon, lat) + lats = lat2d.flatten() + lons = lon2d.flatten() + + state = { + "forecast_reference_time": forecast_reference_time, + "valid_time": valid_time, + "latitudes": lats, + "longitudes": lons, + "fields": {}, + } + + # --- LAM envelope (convex hull) --- + lam_hull = MultiPoint(list(zip(lons.tolist(), lats.tolist()))).convex_hull + state["lam_envelope"] = gpd.GeoSeries([lam_hull], crs="EPSG:4326") + + # --- extract spatial fields --- + for param in paramlist: + # e.g. U_10M.MAE.spatial + matching_vars = [ + v for v in ds.data_vars + if v.startswith(f"{param}.") and v.endswith(".spatial") + ] + + if not matching_vars: + state["fields"][param] = np.full(lats.size, np.nan, dtype=float) + continue + + # If multiple metrics exist, concatenate them + arrays = [] + for var in matching_vars: + arr = ds[var].values # (lead_time, y, x) or (y, x) + arrays.append(arr.reshape(-1)) + + state["fields"][param] = np.concatenate(arrays) + + return state From 35396a70e4b0603bd5b1bb8cdf73fa3610816cab Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 15 Jan 2026 18:06:05 +0100 Subject: [PATCH 018/129] Marimo app cell for loading data from .nc --- src/plotting/compat.py | 1 + workflow/scripts/plot_summary_stat_maps.mo.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/plotting/compat.py b/src/plotting/compat.py index fbe26f3e..06609d1d 100644 --- a/src/plotting/compat.py +++ b/src/plotting/compat.py @@ -5,6 +5,7 @@ import geopandas as gpd import numpy as np import pandas as pd +import xarray as xr from meteodatalab import data_source from meteodatalab import grib_decoder from shapely.geometry import MultiPoint diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 2a60de34..27cc957c 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -84,16 +84,21 @@ def _(ArgumentParser, Path): @app.cell -def _(nc_dir, init_time, lead_time, load_state_from_grib, param): - # load .nc verification file - nc_file = nc_dir / "verif_aggregated.nc" +def _(nc_file, param, lead_time, load_state_from_netcdf): + # load .nc verification file: if param == "SP_10M": paramlist = ["U_10M", "V_10M"] elif param == "SP": paramlist = ["U", "V"] else: paramlist = [param] - state = load_state_from_grib(grib_file, paramlist=paramlist) + + state = load_state_from_netcdf( + nc_file, + paramlist=paramlist, + lead_time=lead_time, + ) + return (state,) From e8a92aaa738e8c5f9e44268057ecf21a0ea4e8d6 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 19 Jan 2026 15:50:02 +0100 Subject: [PATCH 019/129] Remove .nc-loading function again, do it with earthkit instead. --- src/plotting/compat.py | 75 ------------------------------------------ 1 file changed, 75 deletions(-) diff --git a/src/plotting/compat.py b/src/plotting/compat.py index 06609d1d..93634aa3 100644 --- a/src/plotting/compat.py +++ b/src/plotting/compat.py @@ -92,78 +92,3 @@ def load_state_from_raw( state["fields"][key.removeprefix("field_")] = value return state - -def load_state_from_netcdf( - file: Path, - paramlist: list[str], - *, - season: str = "all", - init_hour: int = -999, - lead_time: int | None = None, # hours -) -> dict: - """ - NetCDF analogue of load_state_from_grib(), restricted to spatial variables. - """ - - ds = xr.open_dataset(file) - - # --- normalize lead_time to hours (float) --- - if ds["lead_time"].dtype.kind == "m": - ds = ds.assign_coords( - lead_time=ds["lead_time"].dt.total_seconds() / 3600 - ) - - # --- select season / init_hour / lead_time --- - ds = ds.sel(season=season, init_hour=init_hour) - if lead_time is not None: - ds = ds.sel(lead_time=lead_time) - - # --- infer reference + valid time --- - # Assumption: forecast_reference_time is not explicitly stored - # We reconstruct something consistent with GRIB usage - forecast_reference_time = None - valid_time = None - if lead_time is not None: - valid_time = pd.to_datetime(lead_time, unit="h", origin="unix") - - # --- get lat / lon (assumed present as coordinates) --- - lat = ds["lat"].values if "lat" in ds.coords else ds["latitude"].values - lon = ds["lon"].values if "lon" in ds.coords else ds["longitude"].values - - lon2d, lat2d = np.meshgrid(lon, lat) - lats = lat2d.flatten() - lons = lon2d.flatten() - - state = { - "forecast_reference_time": forecast_reference_time, - "valid_time": valid_time, - "latitudes": lats, - "longitudes": lons, - "fields": {}, - } - - # --- LAM envelope (convex hull) --- - lam_hull = MultiPoint(list(zip(lons.tolist(), lats.tolist()))).convex_hull - state["lam_envelope"] = gpd.GeoSeries([lam_hull], crs="EPSG:4326") - - # --- extract spatial fields --- - for param in paramlist: - # e.g. U_10M.MAE.spatial - matching_vars = [ - v for v in ds.data_vars - if v.startswith(f"{param}.") and v.endswith(".spatial") - ] - - if not matching_vars: - state["fields"][param] = np.full(lats.size, np.nan, dtype=float) - continue - - # If multiple metrics exist, concatenate them - arrays = [] - for var in matching_vars: - arr = ds[var].values # (lead_time, y, x) or (y, x) - arrays.append(arr.reshape(-1)) - - state["fields"][param] = np.concatenate(arrays) - - return state From 06be6fa853dc3536e6271b0bb3d453bcc6149f69 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 21 Jan 2026 18:04:30 +0100 Subject: [PATCH 020/129] All kinds of changes. Co-Development session with Francesco. Got a long way towards the png plots. Co-authored-by: Francesco Zanetta --- workflow/Snakefile | 7 + workflow/rules/plot.smk | 19 +- workflow/scripts/plot_summary_stat_maps.mo.py | 205 +++++------------- 3 files changed, 77 insertions(+), 154 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 7f430cf9..7a6e08b3 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -52,6 +52,13 @@ rule experiment_all: OUT_ROOT / "data/runs/{run_id}/summary.md", run_id=collect_all_candidates(), ), + expand( + OUT_ROOT / "results/experiment/metrics_spatial/{run_id}/{param}_{metric}_{leadtime}.png", + run_id=collect_all_candidates(), + leadtime=[6, 12], + metric=["BIAS", "RMSE", "MAE"], + param=["TOT_PREC", "T_2M"], + ) rule showcase_all: diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 2014e000..6b827dfc 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -70,11 +70,12 @@ rule make_forecast_animation: rule plot_summary_stat_maps: + localrule: True input: script="workflow/scripts/plot_summary_stat_maps.mo.py", - inference_okfile=rules.execute_inference.output.okfile, + verif_file=OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc" output: - OUT_ROOT / "results/summary_stats/maps/{run_id}/{leadtime}/{metric}_{param}_{region}.png", + OUT_ROOT / "results/experiment/metrics_spatial/{run_id}/{param}_{metric}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits resources: @@ -83,18 +84,18 @@ rule plot_summary_stat_maps: runtime="10m", params: nc_out_dir=lambda wc: ( - Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" + Path(OUT_ROOT) / f"data/runs/{wc.run_id}/verif_aggregated.nc" # not sure how to do this, because the baselines are in, e.g., output/data/baselines/COSMO-E/verif_aggregated.nc # and the runs are in output/data/runs/runID/verif_aggregated.nc ).resolve(), shell: """ export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) - python {input.script} \ - --input {params.nc_out_dir} --date {wildcards.init_time} --outfn {output[0]} \ - --param {wildcards.param} --leadtime {wildcards.leadtime} --region {wildcards.region} \ + # python {input.script} \ + # --input {input.verif_file} --outfn {output[0]} \ + # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} # interactive editing (needs to set localrule: True and use only one core) - # marimo edit {input.script} -- \ - # --input {params.grib_out_dir} --date {wildcards.init_time} --outfn {output[0]}\ - # --param {wildcards.param} --leadtime {wildcards.leadtime} --region {wildcards.region}\ + marimo edit {input.script} -- \ + --input {input.verif_file} --outfn {output[0]}\ + --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} """ \ No newline at end of file diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 27cc957c..54bf7863 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -1,12 +1,12 @@ import marimo -__generated_with = "0.16.5" +__generated_with = "0.19.4" app = marimo.App(width="medium") @app.cell def _(): - + # this sure stays the same. import logging from argparse import ArgumentParser @@ -16,10 +16,11 @@ def _(): import cartopy.crs as ccrs import earthkit.plots as ekp import numpy as np + import xarray as xr # this stays the same as well. from plotting import DOMAINS - + # no changes to StatePlotter required according to ChatGPT. from plotting import StatePlotter @@ -28,19 +29,16 @@ def _(): # need to load nc files. But this statement is not needed any more because # the .nc files can just be read with xr.open_dataset - # from plotting.compat import load_state_from_grib - return ( ArgumentParser, CMAP_DEFAULTS, + DOMAINS, Path, StatePlotter, ekp, - # load_state_from_grib, logging, np, - DOMAINS, - ccrs, + xr, ) @@ -49,57 +47,51 @@ def _(logging): LOG = logging.getLogger(__name__) LOG_FMT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" logging.basicConfig(level=logging.INFO, format=LOG_FMT) - return (LOG,) + return @app.cell -def _(ArgumentParser, Path): +def _(ArgumentParser, Path, np): parser = ArgumentParser() parser.add_argument( "--input", type=str, default=None, help="Directory to .nc data containing the error fields" ) - # parser.add_argument("--date", type=str, default=None, help="reference datetime") # to be deleted? parser.add_argument("--outfn", type=str, help="output filename") parser.add_argument("--leadtime", type=str, help="leadtime") parser.add_argument("--param", type=str, help="parameter") - parser.add_argument("--region", type=str, help="name of region") + # parser.add_argument("--region", type=str, help="name of region") + parser.add_argument("--metric", type=str, help = "Evaluation Metric. So far Bias, RMSE or MAE are implemented.") + parser.add_argument("--season", type=str, default="all", help="season filter") + parser.add_argument("--init_hour", type=str, default="all", help="initialization hour filter") args = parser.parse_args() - nc_dir = Path(args.input) - # init_time = args.date # to be deleted? + verif_file = Path(args.input) outfn = Path(args.outfn) lead_time = args.leadtime param = args.param - region = args.region - return ( - args, - nc_dir, - # init_time, # to be deleted? - lead_time, - outfn, - param, - region, - ) + # region = args.region + season = args.season + init_hour = args.init_hour + metric = args.metric + if isinstance(init_hour, str): + if init_hour == "all": + init_hour = -999 + else: + raise ValueError("init_hour must be 'all' or an integer hour") -@app.cell -def _(nc_file, param, lead_time, load_state_from_netcdf): - # load .nc verification file: - if param == "SP_10M": - paramlist = ["U_10M", "V_10M"] - elif param == "SP": - paramlist = ["U", "V"] - else: - paramlist = [param] - - state = load_state_from_netcdf( - nc_file, - paramlist=paramlist, - lead_time=lead_time, - ) + lead_time = np.timedelta64(lead_time, 'h') + return init_hour, lead_time, metric, outfn, param, season, verif_file - return (state,) + +@app.cell +def _(init_hour, lead_time, metric, param, season, verif_file, xr): + ds = xr.open_dataset(verif_file) + var = f"{param}.{metric}.spatial" + ds = ds[var].sel(init_hour=init_hour, lead_time=lead_time, season=season) + ds + return ds, var @app.cell @@ -125,129 +117,52 @@ def get_style(param, units_override=None): "vmax": cfg.get("vmax", None), "colors": cfg.get("colors", None), } - return (get_style,) @app.cell -def _(LOG, np): - """Preprocess fields with pint-based unit conversion and derived quantities.""" - try: - import pint # type: ignore - - _ureg = pint.UnitRegistry() - - def _k_to_c(arr): - # robust conversion with pint, fallback if dtype unsupported - try: - return (_ureg.Quantity(arr, _ureg.kelvin).to(_ureg.degC)).magnitude - except Exception: - return arr - 273.15 - - def _ms_to_knots(arr): - # robust conversion with pint, fallback if dtype unsupported - try: - return ( - _ureg.Quantity(arr, _ureg.meter / _ureg.second).to(_ureg.knot) - ).magnitude - except Exception: - return arr * 1.943844 - - def _m_to_mm(arr): - # robust conversion with pint, fallback if dtype unsupported - try: - return (_ureg.Quantity(arr, _ureg.meter).to(_ureg.millimeter)).magnitude - except Exception: - return arr * 1000 - - except Exception: - LOG.warning("pint not available; falling back hardcoded conversions") - - def _k_to_c(arr): - return arr - 273.15 - - def _ms_to_knots(arr): - return arr * 1.943844 - - def _m_to_mm(arr): - return arr * 1000 - - def preprocess_field(param: str, state: dict): - """ - - Temperatures: K -> °C - - Wind speed: sqrt(u^2 + v^2) - - Precipitation: m -> mm - Returns: (field_array, units_override or None) - """ - fields = state["fields"] - # temperature variables - if param in ("T_2M", "TD_2M", "T", "TD"): - return _k_to_c(fields[param]), "°C" - # 10m wind speed (allow legacy 'uv' alias) - if param == "SP_10M": - u = fields["U_10M"] - v = fields["V_10M"] - return np.sqrt(u**2 + v**2), "m/s" - # wind speed from standard-level components - if param == "SP": - u = fields["U"] - v = fields["V"] - return np.sqrt(u**2 + v**2), "m/s" - if param == "TOT_PREC": - return _m_to_mm(fields[param]), "mm" - # default: passthrough - return fields[param], None - - return (preprocess_field,) - - -@app.cell -def _( - LOG, - StatePlotter, - args, - get_style, - outfn, - param, - preprocess_field, - region, - state, - DOMAINS, - ccrs, -): +def _(DOMAINS, StatePlotter, ds, get_style, outfn, var): # plot individual fields + import matplotlib.pyplot as plt + plotter = StatePlotter( - state["longitudes"], - state["latitudes"], + ds["longitude"].values.ravel(), + ds["latitude"].values.ravel(), outfn.parent, ) fig = plotter.init_geoaxes( nrows=1, ncols=1, - projection=DOMAINS[region]["projection"], - bbox=DOMAINS[region]["extent"], - name=region, + projection=DOMAINS["switzerland"]["projection"], + bbox=DOMAINS["switzerland"]["extent"], + name="switzerland", size=(6, 6), ) subplot = fig.add_map(row=0, column=0) - # preprocess field (unit conversion, derived quantities) - field, units_override = preprocess_field(param, state) + # # preprocess field (unit conversion, derived quantities) + # field, units_override = preprocess_field(param, state) - plotter.plot_field(subplot, field, **get_style(args.param, units_override)) - subplot.ax.add_geometries( - state["lam_envelope"], - edgecolor="black", - facecolor="none", - crs=ccrs.PlateCarree(), - ) - validtime = state["valid_time"].strftime("%Y%m%d%H%M") - # leadtime = int(state["lead_time"].total_seconds() // 3600) + plotter.plot_field(subplot, ds.values.ravel(), **get_style(var)) + # subplot.ax.add_geometries( + # state["lam_envelope"], + # edgecolor="black", + # facecolor="none", + # crs=ccrs.PlateCarree(), + # ) + plt.show() + # validtime = state["valid_time"].strftime("%Y%m%d%H%M") + # # leadtime = int(state["lead_time"].total_seconds() // 3600) + + # fig.title(f"{param}, time: {validtime}") + + # fig.save(outfn, bbox_inches="tight", dpi=200) + # LOG.info(f"saved: {outfn}") + return - fig.title(f"{param}, time: {validtime}") - fig.save(outfn, bbox_inches="tight", dpi=200) - LOG.info(f"saved: {outfn}") +@app.cell +def _(): return From 61c728ec66ea55bfbcbe7164920d44115d590fa0 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 22 Jan 2026 17:14:21 +0100 Subject: [PATCH 021/129] Generalized to the other non-trivial (non-wind) variables. --- workflow/Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 7a6e08b3..41b54415 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -57,7 +57,7 @@ rule experiment_all: run_id=collect_all_candidates(), leadtime=[6, 12], metric=["BIAS", "RMSE", "MAE"], - param=["TOT_PREC", "T_2M"], + param=["TD_2M", "T_2M", "PMSL", "PS", "TOT_PREC"], ) From 761f8d94cff59a721172260f54f5781d4ea64e37 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 22 Jan 2026 17:24:11 +0100 Subject: [PATCH 022/129] Some changes to plotting script. --- workflow/scripts/plot_summary_stat_maps.mo.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 54bf7863..52ea116d 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -26,9 +26,6 @@ def _(): # Added some new colour maps for the Bias / MAE / RMSE map plots. from plotting.colormap_defaults import CMAP_DEFAULTS - - # need to load nc files. But this statement is not needed any more because - # the .nc files can just be read with xr.open_dataset return ( ArgumentParser, CMAP_DEFAULTS, @@ -47,7 +44,7 @@ def _(logging): LOG = logging.getLogger(__name__) LOG_FMT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" logging.basicConfig(level=logging.INFO, format=LOG_FMT) - return + return (LOG,) @app.cell @@ -121,7 +118,18 @@ def get_style(param, units_override=None): @app.cell -def _(DOMAINS, StatePlotter, ds, get_style, outfn, var): +def _( + DOMAINS, + LOG, + StatePlotter, + ds, + get_style, + lead_time, + metric, + outfn, + param, + var, +): # plot individual fields import matplotlib.pyplot as plt @@ -150,11 +158,11 @@ def _(DOMAINS, StatePlotter, ds, get_style, outfn, var): # facecolor="none", # crs=ccrs.PlateCarree(), # ) - plt.show() + # validtime = state["valid_time"].strftime("%Y%m%d%H%M") # # leadtime = int(state["lead_time"].total_seconds() // 3600) - # fig.title(f"{param}, time: {validtime}") + fig.title(f"{metric} of {param}, Lead Time: {lead_time}") # fig.save(outfn, bbox_inches="tight", dpi=200) # LOG.info(f"saved: {outfn}") From 1cd4dbf22e87464b69cd52d7126109d5722abd8d Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 23 Jan 2026 13:25:07 +0100 Subject: [PATCH 023/129] Plotting region now dynamical (but not yet properly working). Output written to .png now working. --- workflow/Snakefile | 3 ++- workflow/rules/plot.smk | 19 +++++++++++-------- workflow/scripts/plot_summary_stat_maps.mo.py | 13 +++++++------ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 41b54415..0c96d3c8 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -53,11 +53,12 @@ rule experiment_all: run_id=collect_all_candidates(), ), expand( - OUT_ROOT / "results/experiment/metrics_spatial/{run_id}/{param}_{metric}_{leadtime}.png", + OUT_ROOT / "results/experiment/metrics_spatial/{run_id}/{param}_{metric}_{leadtime}_{region}.png", run_id=collect_all_candidates(), leadtime=[6, 12], metric=["BIAS", "RMSE", "MAE"], param=["TD_2M", "T_2M", "PMSL", "PS", "TOT_PREC"], + region=["europe", "switzerland"] ) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 6b827dfc..cad66bd5 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -73,9 +73,12 @@ rule plot_summary_stat_maps: localrule: True input: script="workflow/scripts/plot_summary_stat_maps.mo.py", - verif_file=OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc" + verif_file=OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", + # verif_file=EXPERIMENT_PARTICIPANTS.values(), + # copied from rule report_experiment_dashboard - should be correct here, + # but needs adjustments in the output as well - tbd. output: - OUT_ROOT / "results/experiment/metrics_spatial/{run_id}/{param}_{metric}_{leadtime}.png", + OUT_ROOT / "results/experiment/metrics_spatial/{run_id}/{param}_{metric}_{leadtime}_{region}.png", wildcard_constraints: leadtime=r"\d+", # only digits resources: @@ -91,11 +94,11 @@ rule plot_summary_stat_maps: shell: """ export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) - # python {input.script} \ - # --input {input.verif_file} --outfn {output[0]} \ - # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} + python {input.script} \ + --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ + --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ # interactive editing (needs to set localrule: True and use only one core) - marimo edit {input.script} -- \ - --input {input.verif_file} --outfn {output[0]}\ - --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} + # marimo edit {input.script} -- \ + # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ + # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ """ \ No newline at end of file diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 52ea116d..0f68c203 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -57,7 +57,7 @@ def _(ArgumentParser, Path, np): parser.add_argument("--outfn", type=str, help="output filename") parser.add_argument("--leadtime", type=str, help="leadtime") parser.add_argument("--param", type=str, help="parameter") - # parser.add_argument("--region", type=str, help="name of region") + parser.add_argument("--region", type=str, help="name of region") parser.add_argument("--metric", type=str, help = "Evaluation Metric. So far Bias, RMSE or MAE are implemented.") parser.add_argument("--season", type=str, default="all", help="season filter") parser.add_argument("--init_hour", type=str, default="all", help="initialization hour filter") @@ -128,6 +128,7 @@ def _( metric, outfn, param, + region, var, ): # plot individual fields @@ -141,9 +142,9 @@ def _( fig = plotter.init_geoaxes( nrows=1, ncols=1, - projection=DOMAINS["switzerland"]["projection"], - bbox=DOMAINS["switzerland"]["extent"], - name="switzerland", + projection=DOMAINS[region]["projection"], + bbox=DOMAINS[region]["extent"], + name=region, size=(6, 6), ) subplot = fig.add_map(row=0, column=0) @@ -164,8 +165,8 @@ def _( fig.title(f"{metric} of {param}, Lead Time: {lead_time}") - # fig.save(outfn, bbox_inches="tight", dpi=200) - # LOG.info(f"saved: {outfn}") + fig.save(outfn, bbox_inches="tight", dpi=200) + LOG.info(f"saved: {outfn}") return From cdb2ccd8f91890b55386c9dbaabfbbd1b4eb967b Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 23 Jan 2026 13:31:00 +0100 Subject: [PATCH 024/129] Dynamic Regions now working. --- workflow/scripts/plot_summary_stat_maps.mo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 0f68c203..a2be8ca1 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -67,7 +67,7 @@ def _(ArgumentParser, Path, np): outfn = Path(args.outfn) lead_time = args.leadtime param = args.param - # region = args.region + region = args.region season = args.season init_hour = args.init_hour metric = args.metric From 558689c980b886c53d4a99022c477c76e7f14149 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 23 Jan 2026 14:36:07 +0100 Subject: [PATCH 025/129] Store results under experiment hash. --- workflow/Snakefile | 5 +++-- workflow/rules/plot.smk | 5 +---- workflow/scripts/plot_summary_stat_maps.mo.py | 11 ++++++++++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 0c96d3c8..f8c9214f 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -53,12 +53,13 @@ rule experiment_all: run_id=collect_all_candidates(), ), expand( - OUT_ROOT / "results/experiment/metrics_spatial/{run_id}/{param}_{metric}_{leadtime}_{region}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/{run_id}/{param}_{metric}_{leadtime}_{region}.png", run_id=collect_all_candidates(), leadtime=[6, 12], metric=["BIAS", "RMSE", "MAE"], param=["TD_2M", "T_2M", "PMSL", "PS", "TOT_PREC"], - region=["europe", "switzerland"] + region=["europe", "switzerland"], + experiment=EXPERIMENT_HASH ) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index cad66bd5..298784a2 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -74,11 +74,8 @@ rule plot_summary_stat_maps: input: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", - # verif_file=EXPERIMENT_PARTICIPANTS.values(), - # copied from rule report_experiment_dashboard - should be correct here, - # but needs adjustments in the output as well - tbd. output: - OUT_ROOT / "results/experiment/metrics_spatial/{run_id}/{param}_{metric}_{leadtime}_{region}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/{run_id}/{param}_{metric}_{leadtime}_{region}.png", wildcard_constraints: leadtime=r"\d+", # only digits resources: diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index a2be8ca1..49ce4893 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -79,7 +79,16 @@ def _(ArgumentParser, Path, np): raise ValueError("init_hour must be 'all' or an integer hour") lead_time = np.timedelta64(lead_time, 'h') - return init_hour, lead_time, metric, outfn, param, season, verif_file + return ( + init_hour, + lead_time, + metric, + outfn, + param, + region, + season, + verif_file, + ) @app.cell From 986a7ee0ce75ba9bfa1f91407c83de7d579dea80 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 26 Jan 2026 15:05:39 +0100 Subject: [PATCH 026/129] Introduced new domain "switzerland_small" for more detailed inspection of results at smaller spatial scale. --- src/plotting/__init__.py | 4 ++++ workflow/Snakefile | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plotting/__init__.py b/src/plotting/__init__.py index ce5e4e63..520e35ae 100644 --- a/src/plotting/__init__.py +++ b/src/plotting/__init__.py @@ -37,6 +37,10 @@ "extent": [0, 17.5, 40.5, 53.0], "projection": _PROJECTIONS["orthographic"], }, + "switzerland_small": { + "extent": [5.5, 11.0, 45.5, 48.0], + "projection": _PROJECTIONS["orthographic"], + }, } diff --git a/workflow/Snakefile b/workflow/Snakefile index f8c9214f..64a30694 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -58,7 +58,7 @@ rule experiment_all: leadtime=[6, 12], metric=["BIAS", "RMSE", "MAE"], param=["TD_2M", "T_2M", "PMSL", "PS", "TOT_PREC"], - region=["europe", "switzerland"], + region=["switzerland", "switzerland_small"], experiment=EXPERIMENT_HASH ) From 7f70fb453c49340e377e8224ccbeb1fce08b3ce6 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 27 Jan 2026 16:04:46 +0100 Subject: [PATCH 027/129] Reverse Red-Blue colour maps for bias. --- src/plotting/colormap_defaults.py | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 5e22f72f..5539b363 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -11,16 +11,16 @@ def _fallback(): warnings.warn("No colormap found for this parameter, using fallback.", UserWarning) return {"cmap": plt.get_cmap("viridis"), "norm": None, "units": ""} -def symmetric_boundary_norm(nlevels): - """ - Returns a callable that creates a symmetric BoundaryNorm - around zero with `nlevels` discrete colors. Used for creating colormaps for bias. - """ - def _norm(data): - vmax = np.nanmax(np.abs(data)) - boundaries = np.linspace(-vmax, vmax, nlevels + 1) - return BoundaryNorm(boundaries=boundaries, ncolors=nlevels) - return _norm +# def symmetric_boundary_norm(nlevels): +# """ +# Returns a callable that creates a symmetric BoundaryNorm +# around zero with `nlevels` discrete colors. Used for creating colormaps for bias. +# """ +# def _norm(data): +# vmax = np.nanmax(np.abs(data)) +# boundaries = np.linspace(-vmax, vmax, nlevels + 1) +# return BoundaryNorm(boundaries=boundaries, ncolors=nlevels) +# return _norm _CMAP_DEFAULTS = { "SP": {"cmap": plt.get_cmap("coolwarm", 11), "vmin": 800 * 100, "vmax": 1100 * 100}, @@ -83,13 +83,13 @@ def _norm(data): # Bias: # diverging colour scheme for the Bias to reflect the nature of the data (can be positive or negative, symmetric). # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. - "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "m/s"}, - "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "m/s"}, - "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "°C"}, - "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "°C"}, - "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "Pa"}, - "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "Pa"}, - "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 11), "norm": symmetric_boundary_norm(nlevels=11)} | {"units": "mm"} + "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, + "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, + "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "°C"}, + "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "°C"}, + "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, + "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, + "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 11)} | {"units": "mm"} } CMAP_DEFAULTS = defaultdict(_fallback, _CMAP_DEFAULTS) From 50e899fdce05ffe395b98a0d107ff986cee79fe4 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 27 Jan 2026 16:08:51 +0100 Subject: [PATCH 028/129] Preliminary changes to plotting script for getting symmetric colour map for bias. --- workflow/scripts/plot_summary_stat_maps.mo.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 49ce4893..2d81ac8d 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -102,25 +102,38 @@ def _(init_hour, lead_time, metric, param, season, verif_file, xr): @app.cell def _(CMAP_DEFAULTS, ekp): - def get_style(param, units_override=None): + def get_style(param, metric, units_override=None): """Get style and colormap settings for the plot. Needed because cmap/norm does not work in Style(colors=cmap), still needs to be passed as arguments to tripcolor()/tricontourf(). """ cfg = CMAP_DEFAULTS[param] units = units_override if units_override is not None else cfg.get("units", "") + + levels = cfg.get("levels", None) + + if (metric == "BIAS"): + # For bias, we want to use a diverging colormap with symmetric bounds around zero. + max_abs_val = max(abs(levels)) + levels = numpy.arange(-max_abs_val, max_abs_val, dtype=None) + vmin = -max_abs_val + vmax = max_abs_val + else: + vmin = cfg.get("vmin", None) + vmax = cfg.get("vmax", None) + return { "style": ekp.styles.Style( - levels=cfg.get("bounds", cfg.get("levels", None)), + levels=cfg.get("bounds", levels), extend="both", units=units, colors=cfg.get("colors", None), ), "norm": cfg.get("norm", None), "cmap": cfg.get("cmap", None), - "levels": cfg.get("levels", None), - "vmin": cfg.get("vmin", None), - "vmax": cfg.get("vmax", None), + "levels": levels, + "vmin": vmin, + "vmax": vmax, "colors": cfg.get("colors", None), } return (get_style,) @@ -161,7 +174,7 @@ def _( # # preprocess field (unit conversion, derived quantities) # field, units_override = preprocess_field(param, state) - plotter.plot_field(subplot, ds.values.ravel(), **get_style(var)) + plotter.plot_field(subplot, ds.values.ravel(), **get_style(var, metric)) # subplot.ax.add_geometries( # state["lam_envelope"], # edgecolor="black", From a0aefc0ad065e272b4cbc28b41fd21de30f1eb1b Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 27 Jan 2026 16:23:57 +0100 Subject: [PATCH 029/129] Temporarily changed plotting script back to original to see if all of it still works. --- workflow/scripts/plot_summary_stat_maps.mo.py | 62 +++++++++++++------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 2d81ac8d..9b51c920 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -99,45 +99,69 @@ def _(init_hour, lead_time, metric, param, season, verif_file, xr): ds return ds, var - @app.cell def _(CMAP_DEFAULTS, ekp): - def get_style(param, metric, units_override=None): + def get_style(param, units_override=None): """Get style and colormap settings for the plot. Needed because cmap/norm does not work in Style(colors=cmap), still needs to be passed as arguments to tripcolor()/tricontourf(). """ cfg = CMAP_DEFAULTS[param] units = units_override if units_override is not None else cfg.get("units", "") - - levels = cfg.get("levels", None) - - if (metric == "BIAS"): - # For bias, we want to use a diverging colormap with symmetric bounds around zero. - max_abs_val = max(abs(levels)) - levels = numpy.arange(-max_abs_val, max_abs_val, dtype=None) - vmin = -max_abs_val - vmax = max_abs_val - else: - vmin = cfg.get("vmin", None) - vmax = cfg.get("vmax", None) - return { "style": ekp.styles.Style( - levels=cfg.get("bounds", levels), + levels=cfg.get("bounds", cfg.get("levels", None)), extend="both", units=units, colors=cfg.get("colors", None), ), "norm": cfg.get("norm", None), "cmap": cfg.get("cmap", None), - "levels": levels, - "vmin": vmin, - "vmax": vmax, + "levels": cfg.get("levels", None), + "vmin": cfg.get("vmin", None), + "vmax": cfg.get("vmax", None), "colors": cfg.get("colors", None), } + return (get_style,) +# @app.cell +# def _(CMAP_DEFAULTS, ekp): +# def get_style(param, metric, units_override=None): +# """Get style and colormap settings for the plot. +# Needed because cmap/norm does not work in Style(colors=cmap), +# still needs to be passed as arguments to tripcolor()/tricontourf(). +# """ +# cfg = CMAP_DEFAULTS[param] +# units = units_override if units_override is not None else cfg.get("units", "") + +# levels = cfg.get("levels", None) + +# if (metric == "BIAS"): +# # For bias, we want to use a diverging colormap with symmetric bounds around zero. +# max_abs_val = max(abs(levels)) +# levels = numpy.arange(-max_abs_val, max_abs_val, dtype=None) +# vmin = -max_abs_val +# vmax = max_abs_val +# else: +# vmin = cfg.get("vmin", None) +# vmax = cfg.get("vmax", None) + +# return { +# "style": ekp.styles.Style( +# levels=cfg.get("bounds", levels), +# extend="both", +# units=units, +# colors=cfg.get("colors", None), +# ), +# "norm": cfg.get("norm", None), +# "cmap": cfg.get("cmap", None), +# "levels": levels, +# "vmin": vmin, +# "vmax": vmax, +# "colors": cfg.get("colors", None), +# } +# return (get_style,) @app.cell def _( From be6fa354094d9a1c8dc88da10d94b64603b26f7a Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 27 Jan 2026 18:04:52 +0100 Subject: [PATCH 030/129] Fix to the functioning of _compute_scores and _compute_statistics with argument dim. --- src/verification/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index 8f05349c..747d3b4e 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -77,12 +77,13 @@ def _compute_scores( prefix="", suffix="", source="", + dim=["x", "y"], ) -> xr.Dataset: """ Compute basic verification metrics between two xarray DataArrays (fcst and obs). Returns a xarray Dataset with the computed metrics. """ - dim = ["x", "y"] if "x" in fcst.dims and "y" in fcst.dims else ["values"] + error = fcst - obs if dim == []: scores = xr.Dataset( @@ -112,12 +113,13 @@ def _compute_statistics( prefix="", suffix="", source="", + dim=["x", "y"], ) -> xr.Dataset: """ Compute basic statistics of a xarray DataArray (data). Returns a xarray Dataset with the computed statistics. """ - dim = ["x", "y"] if "x" in data.dims and "y" in data.dims else ["values"] + stats = xr.Dataset( { f"{prefix}mean{suffix}": data.mean(dim=dim, skipna=True), @@ -161,6 +163,8 @@ def verify( """ start = time.time() + dim = ["x", "y"] if "x" in fcst.dims and "y" in fcst.dims else ["values"] + # rewrite the verification to use dask and xarray # chunk the data to avoid memory issues # compute the metrics in parallel @@ -188,19 +192,19 @@ def verify( # scores vs time (reduce spatially) score.append( _compute_scores( - fcst_param, obs_param, prefix=param + ".", source=fcst_label + fcst_param, obs_param, prefix=param + ".", source=fcst_label, dim=dim ).expand_dims(region=[region]) ) # statistics vs time (reduce spatially) fcst_statistics.append( _compute_statistics( - fcst_param, prefix=param + ".", source=fcst_label + fcst_param, prefix=param + ".", source=fcst_label, dim=dim ).expand_dims(region=[region]) ) obs_statistics.append( _compute_statistics( - obs_param, prefix=param + ".", source=obs_label + obs_param, prefix=param + ".", source=obs_label, dim=dim ).expand_dims(region=[region]) ) From 2fcd1f92a0ecd0ae8de2644ea1b9d3b04a90713a Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 29 Jan 2026 15:32:25 +0100 Subject: [PATCH 031/129] Working version for colour scale for bias that is symmetric about zero. vmin and vmax values for variables other than T_2M yet to be defined. --- src/plotting/colormap_defaults.py | 13 +--- workflow/scripts/plot_summary_stat_maps.mo.py | 63 ++++++------------- 2 files changed, 19 insertions(+), 57 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 5539b363..10c461a4 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -11,17 +11,6 @@ def _fallback(): warnings.warn("No colormap found for this parameter, using fallback.", UserWarning) return {"cmap": plt.get_cmap("viridis"), "norm": None, "units": ""} -# def symmetric_boundary_norm(nlevels): -# """ -# Returns a callable that creates a symmetric BoundaryNorm -# around zero with `nlevels` discrete colors. Used for creating colormaps for bias. -# """ -# def _norm(data): -# vmax = np.nanmax(np.abs(data)) -# boundaries = np.linspace(-vmax, vmax, nlevels + 1) -# return BoundaryNorm(boundaries=boundaries, ncolors=nlevels) -# return _norm - _CMAP_DEFAULTS = { "SP": {"cmap": plt.get_cmap("coolwarm", 11), "vmin": 800 * 100, "vmax": 1100 * 100}, "TD_2M": load_ncl_colormap("t2m_29lev.ct"), @@ -86,7 +75,7 @@ def _fallback(): "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "°C"}, - "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "°C"}, + "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "vmin": -5.5, "vmax": 5.5} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 11)} | {"units": "mm"} diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 9b51c920..349ea300 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -100,68 +100,41 @@ def _(init_hour, lead_time, metric, param, season, verif_file, xr): return ds, var @app.cell -def _(CMAP_DEFAULTS, ekp): - def get_style(param, units_override=None): +def _(CMAP_DEFAULTS, ekp, np): + def get_style(param, metric, units_override=None): """Get style and colormap settings for the plot. Needed because cmap/norm does not work in Style(colors=cmap), still needs to be passed as arguments to tripcolor()/tricontourf(). """ - cfg = CMAP_DEFAULTS[param] + + metric_key = f"{param}.{metric}.spatial" + cfg = CMAP_DEFAULTS[metric_key] if metric_key in CMAP_DEFAULTS else CMAP_DEFAULTS[param] units = units_override if units_override is not None else cfg.get("units", "") + + vmin = cfg.get("vmin", None) + vmax = cfg.get("vmax", None) + levels = cfg.get("levels", None) + + # For the case of Bias, construct symmetric levels: + if metric == "BIAS": + levels = np.linspace(start = vmin, stop = vmax, num=12) + return { "style": ekp.styles.Style( - levels=cfg.get("bounds", cfg.get("levels", None)), + levels=cfg.get("bounds", levels), extend="both", units=units, colors=cfg.get("colors", None), ), "norm": cfg.get("norm", None), "cmap": cfg.get("cmap", None), - "levels": cfg.get("levels", None), - "vmin": cfg.get("vmin", None), - "vmax": cfg.get("vmax", None), + "levels": levels, + "vmin": vmin, + "vmax": vmax, "colors": cfg.get("colors", None), } - return (get_style,) -# @app.cell -# def _(CMAP_DEFAULTS, ekp): -# def get_style(param, metric, units_override=None): -# """Get style and colormap settings for the plot. -# Needed because cmap/norm does not work in Style(colors=cmap), -# still needs to be passed as arguments to tripcolor()/tricontourf(). -# """ -# cfg = CMAP_DEFAULTS[param] -# units = units_override if units_override is not None else cfg.get("units", "") - -# levels = cfg.get("levels", None) - -# if (metric == "BIAS"): -# # For bias, we want to use a diverging colormap with symmetric bounds around zero. -# max_abs_val = max(abs(levels)) -# levels = numpy.arange(-max_abs_val, max_abs_val, dtype=None) -# vmin = -max_abs_val -# vmax = max_abs_val -# else: -# vmin = cfg.get("vmin", None) -# vmax = cfg.get("vmax", None) - -# return { -# "style": ekp.styles.Style( -# levels=cfg.get("bounds", levels), -# extend="both", -# units=units, -# colors=cfg.get("colors", None), -# ), -# "norm": cfg.get("norm", None), -# "cmap": cfg.get("cmap", None), -# "levels": levels, -# "vmin": vmin, -# "vmax": vmax, -# "colors": cfg.get("colors", None), -# } -# return (get_style,) @app.cell def _( From 7864b5d3d68a1541be11bcb951d1856b20004616 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 29 Jan 2026 17:21:22 +0100 Subject: [PATCH 032/129] New colour map defaults for T_2M --- src/plotting/colormap_defaults.py | 28 +++++++++---------- workflow/scripts/plot_summary_stat_maps.mo.py | 4 +++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 10c461a4..e3ed2e22 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -52,22 +52,22 @@ def _fallback(): # always start at 0 so that the saturation of the colour corresponds to the error magnitude. # RMSE: - "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "m/s"}, - "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "m/s"}, - "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "°C"}, - "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "°C"}, - "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "Pa"}, - "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "Pa"}, - "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "mm"}, + "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, + "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0, "vmax": 6} | {"units": "°C"}, + "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, + "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, + "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, # MAE: - "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "m/s"}, - "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "m/s"}, - "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "°C"}, - "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "°C"}, - "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "Pa"}, - "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "Pa"}, - "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 11), "vmin": 0} | {"units": "mm"}, + "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, + "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0, "vmax": 6} | {"units": "°C"}, + "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, + "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, + "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, # Bias: # diverging colour scheme for the Bias to reflect the nature of the data (can be positive or negative, symmetric). diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 349ea300..af4be700 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -118,6 +118,10 @@ def get_style(param, metric, units_override=None): # For the case of Bias, construct symmetric levels: if metric == "BIAS": levels = np.linspace(start = vmin, stop = vmax, num=12) + elif metric == "RMSE": + levels = np.linspace(start = vmin, stop = vmax, num=7) + elif metric == "MAE": + levels = np.linspace(start = vmin, stop = vmax, num=7) return { "style": ekp.styles.Style( From de0f2794ed845e44d243b54fec44a3fdc5247576 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 3 Feb 2026 16:26:46 +0100 Subject: [PATCH 033/129] Hard-Code levels for colour breaks. This way, everything is in the same place and can be changed there when needed. Accompanying changes in the plotting script (marimo cell that gets the colour map defaults). --- src/plotting/colormap_defaults.py | 6 ++--- workflow/scripts/plot_summary_stat_maps.mo.py | 24 ++++--------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index e3ed2e22..c9968240 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -55,7 +55,7 @@ def _fallback(): "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, - "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0, "vmax": 6} | {"units": "°C"}, + "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 2, 3, 4, 5, 6]} | {"units": "°C"}, "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, @@ -64,7 +64,7 @@ def _fallback(): "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, - "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0, "vmax": 6} | {"units": "°C"}, + "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 2, 3, 4, 5, 6]} | {"units": "°C"}, "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, @@ -75,7 +75,7 @@ def _fallback(): "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "°C"}, - "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "vmin": -5.5, "vmax": 5.5} | {"units": "°C"}, + "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : [-5.5, -4.5, -3.5, -2.5, -1.5, -0.5, 0.5, 1.5, 2.5, 3.5, 4.5, 5.5]} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 11)} | {"units": "mm"} diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index af4be700..4ee6e9da 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -100,41 +100,27 @@ def _(init_hour, lead_time, metric, param, season, verif_file, xr): return ds, var @app.cell -def _(CMAP_DEFAULTS, ekp, np): +def _(CMAP_DEFAULTS, ekp): def get_style(param, metric, units_override=None): """Get style and colormap settings for the plot. Needed because cmap/norm does not work in Style(colors=cmap), still needs to be passed as arguments to tripcolor()/tricontourf(). """ - metric_key = f"{param}.{metric}.spatial" cfg = CMAP_DEFAULTS[metric_key] if metric_key in CMAP_DEFAULTS else CMAP_DEFAULTS[param] units = units_override if units_override is not None else cfg.get("units", "") - - vmin = cfg.get("vmin", None) - vmax = cfg.get("vmax", None) - levels = cfg.get("levels", None) - - # For the case of Bias, construct symmetric levels: - if metric == "BIAS": - levels = np.linspace(start = vmin, stop = vmax, num=12) - elif metric == "RMSE": - levels = np.linspace(start = vmin, stop = vmax, num=7) - elif metric == "MAE": - levels = np.linspace(start = vmin, stop = vmax, num=7) - return { "style": ekp.styles.Style( - levels=cfg.get("bounds", levels), + levels=cfg.get("bounds", cfg.get("levels", None)), extend="both", units=units, colors=cfg.get("colors", None), ), "norm": cfg.get("norm", None), "cmap": cfg.get("cmap", None), - "levels": levels, - "vmin": vmin, - "vmax": vmax, + "levels": cfg.get("levels", None), + "vmin": cfg.get("vmin", None), + "vmax": cfg.get("vmax", None), "colors": cfg.get("colors", None), } return (get_style,) From 69e7b36dce3537ff12c2986e6b0f3d853f5a5967 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 3 Feb 2026 17:18:29 +0100 Subject: [PATCH 034/129] New Rule for Map plots of Baselines. --- workflow/rules/plot.smk | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 298784a2..ee113266 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -75,7 +75,7 @@ rule plot_summary_stat_maps: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", output: - OUT_ROOT / "results/{experiment}/metrics/spatial/{run_id}/{param}_{metric}_{leadtime}_{region}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/{run_id}/{param}_{metric}_{region}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits resources: @@ -98,4 +98,16 @@ rule plot_summary_stat_maps: # marimo edit {input.script} -- \ # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ - """ \ No newline at end of file + """ + +use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: + input: + script="workflow/scripts/plot_summary_stat_maps.mo.py", + verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", + output: + OUT_ROOT / "results/{experiment}/metrics/spatial/{baseline_id}/{param}_{metric}_{region}_{leadtime}.png", + params: + nc_out_dir=lambda wc: ( + Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" + # not sure if this is actually needed. Verification file is already specified above as input. Leave it for the time being. + ).resolve() \ No newline at end of file From 1353788e151fdcedafca79f786bdc29a30cca03a Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 4 Feb 2026 15:14:09 +0100 Subject: [PATCH 035/129] Different File naming (Region first -> Order) Required accompanying change in Snakefile. --- workflow/Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 64a30694..d21469bb 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -53,7 +53,7 @@ rule experiment_all: run_id=collect_all_candidates(), ), expand( - OUT_ROOT / "results/{experiment}/metrics/spatial/{run_id}/{param}_{metric}_{leadtime}_{region}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/{run_id}/{param}_{metric}_{region}_{leadtime}.png", run_id=collect_all_candidates(), leadtime=[6, 12], metric=["BIAS", "RMSE", "MAE"], From 569759a1f46a6730cf8ac7843025d2bb182d5b48 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 5 Feb 2026 16:00:40 +0100 Subject: [PATCH 036/129] Temporarily remove rule for maps for baselines. --- workflow/rules/plot.smk | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index ee113266..54977ac8 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -100,14 +100,14 @@ rule plot_summary_stat_maps: # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ """ -use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: - input: - script="workflow/scripts/plot_summary_stat_maps.mo.py", - verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", - output: - OUT_ROOT / "results/{experiment}/metrics/spatial/{baseline_id}/{param}_{metric}_{region}_{leadtime}.png", - params: - nc_out_dir=lambda wc: ( - Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" - # not sure if this is actually needed. Verification file is already specified above as input. Leave it for the time being. - ).resolve() \ No newline at end of file +# use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: +# input: +# script="workflow/scripts/plot_summary_stat_maps.mo.py", +# verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", +# output: +# OUT_ROOT / "results/{experiment}/metrics/spatial/{baseline_id}/{param}_{metric}_{region}_{leadtime}.png", +# params: +# nc_out_dir=lambda wc: ( +# Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" +# # not sure if this is actually needed. Verification file is already specified above as input. Leave it for the time being. +# ).resolve() \ No newline at end of file From bda0e0d51a2f1d8d0adb104a64d37493eed976a3 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 5 Feb 2026 16:03:04 +0100 Subject: [PATCH 037/129] Renamed domain "switzerland_small". This has caused problems before. --- src/plotting/__init__.py | 2 +- workflow/Snakefile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plotting/__init__.py b/src/plotting/__init__.py index 520e35ae..cb808df8 100644 --- a/src/plotting/__init__.py +++ b/src/plotting/__init__.py @@ -37,7 +37,7 @@ "extent": [0, 17.5, 40.5, 53.0], "projection": _PROJECTIONS["orthographic"], }, - "switzerland_small": { + "switzerlandsmall": { "extent": [5.5, 11.0, 45.5, 48.0], "projection": _PROJECTIONS["orthographic"], }, diff --git a/workflow/Snakefile b/workflow/Snakefile index d21469bb..3bcee554 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -56,9 +56,9 @@ rule experiment_all: OUT_ROOT / "results/{experiment}/metrics/spatial/{run_id}/{param}_{metric}_{region}_{leadtime}.png", run_id=collect_all_candidates(), leadtime=[6, 12], - metric=["BIAS", "RMSE", "MAE"], - param=["TD_2M", "T_2M", "PMSL", "PS", "TOT_PREC"], - region=["switzerland", "switzerland_small"], + metric=["BIAS", "RMSE"], + param=["T_2M", "TOT_PREC"], + region=["switzerland", "switzerlandsmall"], experiment=EXPERIMENT_HASH ) From ccf6279e50ac762a55055c35946fb56875fd6cb1 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 5 Feb 2026 16:04:24 +0100 Subject: [PATCH 038/129] Generate Colour breaks more elegantly. --- src/plotting/colormap_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index c9968240..dd6530a3 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -75,7 +75,7 @@ def _fallback(): "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "°C"}, - "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : [-5.5, -4.5, -3.5, -2.5, -1.5, -0.5, 0.5, 1.5, 2.5, 3.5, 4.5, 5.5]} | {"units": "°C"}, + "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -5.5, stop = 6, step = 1)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 11)} | {"units": "mm"} From 258b2dea37c0376944cf92dcb86f48c1ccc359ab Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 5 Feb 2026 16:24:41 +0100 Subject: [PATCH 039/129] Colour breaks from function for all T2m metrics. --- src/plotting/colormap_defaults.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index dd6530a3..d7b35745 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -55,7 +55,7 @@ def _fallback(): "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, - "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 2, 3, 4, 5, 6]} | {"units": "°C"}, + "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 6.1, step = 1)} | {"units": "°C"}, "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, @@ -64,7 +64,7 @@ def _fallback(): "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, - "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 2, 3, 4, 5, 6]} | {"units": "°C"}, + "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 6.1, step = 1)} | {"units": "°C"}, "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, @@ -75,7 +75,7 @@ def _fallback(): "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "°C"}, - "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -5.5, stop = 6, step = 1)} | {"units": "°C"}, + "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -5.5, stop = 5.6, step = 1)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 11)} | {"units": "mm"} From 5c0151b8e78d9573ea1eabffc8054de285dfdbef Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 9 Feb 2026 19:09:45 +0100 Subject: [PATCH 040/129] Map plots now also for baselines. --- workflow/Snakefile | 11 ++++++++++- workflow/rules/plot.smk | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 3bcee554..991c288d 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -53,13 +53,22 @@ rule experiment_all: run_id=collect_all_candidates(), ), expand( - OUT_ROOT / "results/{experiment}/metrics/spatial/{run_id}/{param}_{metric}_{region}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{leadtime}.png", run_id=collect_all_candidates(), leadtime=[6, 12], metric=["BIAS", "RMSE"], param=["T_2M", "TOT_PREC"], region=["switzerland", "switzerlandsmall"], experiment=EXPERIMENT_HASH + ), + expand( + OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{leadtime}.png", + baseline_id=collect_all_baselines(), + leadtime=[6, 12], + metric=["BIAS", "RMSE"], + param=["T_2M", "TOT_PREC"], + region=["switzerland", "switzerlandsmall"], + experiment=EXPERIMENT_HASH ) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 54977ac8..0190761c 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -75,7 +75,7 @@ rule plot_summary_stat_maps: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", output: - OUT_ROOT / "results/{experiment}/metrics/spatial/{run_id}/{param}_{metric}_{region}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits resources: @@ -100,6 +100,37 @@ rule plot_summary_stat_maps: # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ """ +rule plot_summary_stat_maps_baseline: + localrule: True + input: + script="workflow/scripts/plot_summary_stat_maps.mo.py", + verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", + output: + OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{leadtime}.png", + wildcard_constraints: + leadtime=r"\d+", # only digits + resources: + slurm_partition="postproc", + cpus_per_task=1, + runtime="10m", + params: + nc_out_dir=lambda wc: ( + Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" + # not sure how to do this, because the baselines are in, e.g., output/data/baselines/COSMO-E/verif_aggregated.nc + # and the runs are in output/data/runs/runID/verif_aggregated.nc + ).resolve(), + shell: + """ + export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) + python {input.script} \ + --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ + --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ + # interactive editing (needs to set localrule: True and use only one core) + # marimo edit {input.script} -- \ + # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ + # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ + """ + # use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: # input: # script="workflow/scripts/plot_summary_stat_maps.mo.py", From 7b50809e8cc9fdcd4e684cfed1dd90196ea0235a Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 9 Feb 2026 19:10:28 +0100 Subject: [PATCH 041/129] Fix memory leak occurring during creation of html. --- .../scripts/report_experiment_dashboard.py | 68 ++++++++++++++++++- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/workflow/scripts/report_experiment_dashboard.py b/workflow/scripts/report_experiment_dashboard.py index 327547b0..fc6ed6a9 100644 --- a/workflow/scripts/report_experiment_dashboard.py +++ b/workflow/scripts/report_experiment_dashboard.py @@ -28,19 +28,81 @@ def program_summary_log(args): LOG.info("=" * 80) +def dataset_metrics_to_dataframe( + ds: xr.Dataset, + forbidden_dims=("values",), + metric_dims=("source", "season", "init_hour", "region", "lead_time", "eps"), +): + """ + Convert a verification xarray.Dataset to a pandas.DataFrame compatible with + the old `to_array("stack").to_dataframe()` workflow. + + Returns columns: + - metric_dims... + - stack (e.g. "T_2M.MAE") + - value + """ + + # keep only non-spatial metric variables + metric_vars = [] + for v in ds.data_vars: + da = ds[v] + + # skip spatial metrics + if "spatial" in v: + continue + + # skip forbidden dimensions + if any(d in da.dims for d in forbidden_dims): + continue + + # only keep vars whose dims are a subset of expected metric dims + if not set(da.dims).issubset(metric_dims): + continue + + metric_vars.append(v) + + if not metric_vars: + raise ValueError("No compatible metric variables found in dataset") + + # stack variables exactly like the original code + df = ( + ds[metric_vars] + .to_array("stack") + .to_dataframe(name="value") + .reset_index() + ) + + return df + def main(args): program_summary_log(args) # Load, de-duplicate lead_time, and keep best provider per source (same logic as verif_plot_metrics) - dfs = [xr.open_dataset(f) for f in args.verif_files] + drop_variables = ["TOT_PREC.MAE.spatial", "TOT_PREC.RMSE.spatial", "TOT_PREC.BIAS.spatial", + "PMSL.MAE.spatial", "PMSL.RMSE.spatial", "PMSL.BIAS.spatial", + "PS.MAE.spatial", "PS.RMSE.spatial", "PS.BIAS.spatial", + "V_10M.MAE.spatial", "V_10M.RMSE.spatial", "V_10M.BIAS.spatial", + "U_10M.MAE.spatial", "U_10M.RMSE.spatial", "U_10M.BIAS.spatial", + "TD_2M.MAE.spatial", "TD_2M.RMSE.spatial", "TD_2M.BIAS.spatial", + "T_2M.MAE.spatial", "T_2M.RMSE.spatial", "T_2M.BIAS.spatial"] + + dfs = [xr.open_dataset(f, drop_variables = drop_variables) for f in args.verif_files] dfs = [_ensure_unique_lead_time(d) for d in dfs] dfs = _select_best_sources(dfs) ds = xr.concat(dfs, dim="source", join="outer") LOG.info("Loaded verification netcdf: \n%s", ds) # extract only non-spatial variables to pd.DataFrame - nonspatial_vars = [d for d in ds.data_vars if "spatial" not in d] - df = ds[nonspatial_vars].to_array("stack").to_dataframe(name="value").reset_index() + # nonspatial_vars = [d for d in ds.data_vars if "spatial" not in d] + # df = ds[nonspatial_vars].to_array("stack").to_dataframe(name="value").reset_index() + + df = dataset_metrics_to_dataframe( + ds, + forbidden_dims=("values",), # critical! + metric_dims=("source", "season", "init_hour", "region", "lead_time", "eps"), + ) + df[["param", "metric"]] = df["stack"].str.split(".", n=1, expand=True) df.drop(columns=["stack"], inplace=True) df["lead_time"] = df["lead_time"].dt.total_seconds() / 3600 From c3d37a20fd3db2ff5815003e462ca623054dcac3 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 10 Feb 2026 17:01:08 +0100 Subject: [PATCH 042/129] "Alias" Rule for Plotting maps of Baselines This does the same thing with less code. --- workflow/rules/plot.smk | 37 +++---------------------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 0190761c..2924201e 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -100,45 +100,14 @@ rule plot_summary_stat_maps: # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ """ -rule plot_summary_stat_maps_baseline: - localrule: True +use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: input: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", output: OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{leadtime}.png", - wildcard_constraints: - leadtime=r"\d+", # only digits - resources: - slurm_partition="postproc", - cpus_per_task=1, - runtime="10m", params: nc_out_dir=lambda wc: ( Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" - # not sure how to do this, because the baselines are in, e.g., output/data/baselines/COSMO-E/verif_aggregated.nc - # and the runs are in output/data/runs/runID/verif_aggregated.nc - ).resolve(), - shell: - """ - export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) - python {input.script} \ - --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ - --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ - # interactive editing (needs to set localrule: True and use only one core) - # marimo edit {input.script} -- \ - # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ - # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ - """ - -# use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: -# input: -# script="workflow/scripts/plot_summary_stat_maps.mo.py", -# verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", -# output: -# OUT_ROOT / "results/{experiment}/metrics/spatial/{baseline_id}/{param}_{metric}_{region}_{leadtime}.png", -# params: -# nc_out_dir=lambda wc: ( -# Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" -# # not sure if this is actually needed. Verification file is already specified above as input. Leave it for the time being. -# ).resolve() \ No newline at end of file + # not sure if this is actually needed. Verification file is already specified above as input. Leave it for the time being. + ).resolve() \ No newline at end of file From 8323fbda7017e29b80d46cdba5831d1923332a15 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 11 Feb 2026 10:19:44 +0100 Subject: [PATCH 043/129] Reorganisation of Domains. --- src/plotting/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plotting/__init__.py b/src/plotting/__init__.py index cb808df8..c905fcc3 100644 --- a/src/plotting/__init__.py +++ b/src/plotting/__init__.py @@ -29,15 +29,15 @@ "extent": [-16.0, 25.0, 30.0, 65.0], "projection": _PROJECTIONS["orthographic"], }, + # The domains which are originally called "centraleurope" and "switzerland" + # are mostly the same. I suggest making domain "switzerland" much smaller, + # so that more spatial detail can be seen, especially in the complex + # topography of the alps. "centraleurope": { "extent": [-2.6, 19.5, 40.2, 52.3], "projection": _PROJECTIONS["orthographic"], }, "switzerland": { - "extent": [0, 17.5, 40.5, 53.0], - "projection": _PROJECTIONS["orthographic"], - }, - "switzerlandsmall": { "extent": [5.5, 11.0, 45.5, 48.0], "projection": _PROJECTIONS["orthographic"], }, From d03781cf86b0f558097b688f0effbc5cb6fef363 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 11 Feb 2026 11:57:17 +0100 Subject: [PATCH 044/129] Introduce Seasonal Stratification. --- workflow/Snakefile | 22 ++++++++++--------- workflow/rules/plot.smk | 6 +++-- workflow/scripts/plot_summary_stat_maps.mo.py | 3 ++- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 991c288d..51d13b41 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -53,21 +53,23 @@ rule experiment_all: run_id=collect_all_candidates(), ), expand( - OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", run_id=collect_all_candidates(), - leadtime=[6, 12], - metric=["BIAS", "RMSE"], - param=["T_2M", "TOT_PREC"], - region=["switzerland", "switzerlandsmall"], + leadtime=[6], + metric=["BIAS"], + param=["T_2M"], + region=["centraleurope", "switzerland"], + season=["all", "DJF", "MAM", "JJA", "SON"], experiment=EXPERIMENT_HASH ), expand( - OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", baseline_id=collect_all_baselines(), - leadtime=[6, 12], - metric=["BIAS", "RMSE"], - param=["T_2M", "TOT_PREC"], - region=["switzerland", "switzerlandsmall"], + leadtime=[6], + metric=["BIAS"], + param=["T_2M"], + region=["centraleurope", "switzerland"], + season=["all", "DJF", "MAM", "JJA", "SON"], experiment=EXPERIMENT_HASH ) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 2924201e..10f85c47 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -75,7 +75,7 @@ rule plot_summary_stat_maps: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", output: - OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits resources: @@ -94,10 +94,12 @@ rule plot_summary_stat_maps: python {input.script} \ --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ + --season {wildcards.season} \ # interactive editing (needs to set localrule: True and use only one core) # marimo edit {input.script} -- \ # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ + # --season {wildcards.season} \ """ use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: @@ -105,7 +107,7 @@ use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", output: - OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", params: nc_out_dir=lambda wc: ( Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 4ee6e9da..4ff57749 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -138,6 +138,7 @@ def _( outfn, param, region, + season, var, ): # plot individual fields @@ -172,7 +173,7 @@ def _( # validtime = state["valid_time"].strftime("%Y%m%d%H%M") # # leadtime = int(state["lead_time"].total_seconds() // 3600) - fig.title(f"{metric} of {param}, Lead Time: {lead_time}") + fig.title(f"{metric} of {param}, Season: {season}, Lead Time: {lead_time}") fig.save(outfn, bbox_inches="tight", dpi=200) LOG.info(f"saved: {outfn}") From 48e67892cb1d690aa16b31951d50c08da24db5a9 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 12 Feb 2026 12:33:04 +0100 Subject: [PATCH 045/129] Definitive Colour Levels for 2m-Temperature --- src/plotting/colormap_defaults.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index d7b35745..12dd9420 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -55,7 +55,7 @@ def _fallback(): "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, - "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 6.1, step = 1)} | {"units": "°C"}, + "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 3.1, step = 0.5)} | {"units": "°C"}, "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, @@ -64,7 +64,7 @@ def _fallback(): "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, - "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 6.1, step = 1)} | {"units": "°C"}, + "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 3.1, step = 0.5)} | {"units": "°C"}, "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, @@ -75,7 +75,7 @@ def _fallback(): "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "°C"}, - "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -5.5, stop = 5.6, step = 1)} | {"units": "°C"}, + "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 11)} | {"units": "mm"} From 1c3e62984566d039b7af8ae0ba47b81aa6c53184 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 13 Feb 2026 16:17:09 +0100 Subject: [PATCH 046/129] Colour levels for Precipitation complete. --- src/plotting/colormap_defaults.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 12dd9420..bda0d86c 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -58,8 +58,10 @@ def _fallback(): "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 3.1, step = 0.5)} | {"units": "°C"}, "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, - "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, - + "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.001, 0.0015, 0.002, 0.003, 0.004]} | {"units": "mm"}, + # would ideally want a 6th colour on the high end of the colour scale, but somehow + # matplotlib does not do that -> ? + # MAE: "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, @@ -67,7 +69,8 @@ def _fallback(): "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 3.1, step = 0.5)} | {"units": "°C"}, "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, - "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "mm"}, + "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.001, 0.0015, 0.002, 0.003, 0.004]} | {"units": "mm"}, + # the levels for precip are a bit on the bright side, but still worth keeping consistent with RMSE. # Bias: # diverging colour scheme for the Bias to reflect the nature of the data (can be positive or negative, symmetric). @@ -78,7 +81,7 @@ def _fallback(): "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, - "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 11)} | {"units": "mm"} + "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 9), "levels": [-0.001, -0.0005, -0.00025, -0.0001, 0.0001, 0.00025, 0.0005, 0.001]} | {"units": "mm"} } CMAP_DEFAULTS = defaultdict(_fallback, _CMAP_DEFAULTS) From a4a5607fb4454b382dc3ffe58adbd9cb67a38b07 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 17 Feb 2026 12:14:42 +0100 Subject: [PATCH 047/129] Colour Levels for Dew-Point Temperature and similar style for MAE and RMSE of T2m. --- src/plotting/colormap_defaults.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index bda0d86c..612265b9 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -54,8 +54,8 @@ def _fallback(): # RMSE: "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, - "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, - "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 3.1, step = 0.5)} | {"units": "°C"}, + "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.001, 0.0015, 0.002, 0.003, 0.004]} | {"units": "mm"}, @@ -65,8 +65,8 @@ def _fallback(): # MAE: "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, - "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "°C"}, - "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": np.arange(start = 0, stop = 3.1, step = 0.5)} | {"units": "°C"}, + "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.001, 0.0015, 0.002, 0.003, 0.004]} | {"units": "mm"}, @@ -77,7 +77,7 @@ def _fallback(): # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, - "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "°C"}, + "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, From f245ea38785ad2901746dfd4baf68051d26c9484 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 19 Feb 2026 15:46:58 +0100 Subject: [PATCH 048/129] Calculate and Evaluate Wind Speed too --- src/plotting/colormap_defaults.py | 3 +++ workflow/Snakefile | 4 ++-- workflow/scripts/verif_single_init.py | 11 +++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 612265b9..a9cd0b3d 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -54,6 +54,7 @@ def _fallback(): # RMSE: "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "WS_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, @@ -65,6 +66,7 @@ def _fallback(): # MAE: "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "WS_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, @@ -77,6 +79,7 @@ def _fallback(): # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, + "WS_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, diff --git a/workflow/Snakefile b/workflow/Snakefile index 51d13b41..a2989eb5 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -57,7 +57,7 @@ rule experiment_all: run_id=collect_all_candidates(), leadtime=[6], metric=["BIAS"], - param=["T_2M"], + param=["WS_10M"], region=["centraleurope", "switzerland"], season=["all", "DJF", "MAM", "JJA", "SON"], experiment=EXPERIMENT_HASH @@ -67,7 +67,7 @@ rule experiment_all: baseline_id=collect_all_baselines(), leadtime=[6], metric=["BIAS"], - param=["T_2M"], + param=["WS_10M"], region=["centraleurope", "switzerland"], season=["all", "DJF", "MAM", "JJA", "SON"], experiment=EXPERIMENT_HASH diff --git a/workflow/scripts/verif_single_init.py b/workflow/scripts/verif_single_init.py index 52d9e08f..44cb2ddb 100644 --- a/workflow/scripts/verif_single_init.py +++ b/workflow/scripts/verif_single_init.py @@ -111,8 +111,15 @@ def main(args: ScriptConfig): analysis, ) - # compute metrics and statistics + # before verifying, calculate wind speed: + for ds in [fcst, analysis]: + if "U_10M" in ds and "V_10M" in ds: + LOG.info("Calculating Wind Speed (WS_10M)...") + ds["WS_10M"] = (ds["U_10M"]**2 + ds["V_10M"]**2)**0.5 + # Optional: Add metadata for the netCDF output + ds["WS_10M"].attrs = {"units": "m/s", "long_name": "10m Wind Speed"} + # compute metrics and statistics results = verify(fcst, analysis, args.label, args.analysis_label, args.regions) LOG.info("Verification results:\n%s", results) @@ -150,7 +157,7 @@ def main(args: ScriptConfig): parser.add_argument( "--params", type=lambda x: x.split(","), - default=["T_2M", "TD_2M", "U_10M", "V_10M", "PS", "PMSL", "TOT_PREC"], + default=["T_2M", "TD_2M", "U_10M", "V_10M", "WS_10M", "PS", "PMSL", "TOT_PREC"], ) parser.add_argument( "--steps", From ccea8dbca6de9b20c978d8ed6f75fbe0b625a1e2 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 19 Feb 2026 16:55:34 +0100 Subject: [PATCH 049/129] Fix for Wind speed calculation Now works! --- workflow/scripts/verif_single_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/scripts/verif_single_init.py b/workflow/scripts/verif_single_init.py index 44cb2ddb..27b9cbf6 100644 --- a/workflow/scripts/verif_single_init.py +++ b/workflow/scripts/verif_single_init.py @@ -157,7 +157,7 @@ def main(args: ScriptConfig): parser.add_argument( "--params", type=lambda x: x.split(","), - default=["T_2M", "TD_2M", "U_10M", "V_10M", "WS_10M", "PS", "PMSL", "TOT_PREC"], + default=["T_2M", "TD_2M", "U_10M", "V_10M", "PS", "PMSL", "TOT_PREC"], ) parser.add_argument( "--steps", From ce4242fd5cc14b1b8e0426b4cd773fc240ff675c Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 23 Feb 2026 14:39:12 +0100 Subject: [PATCH 050/129] Name Wind Speed consistent with the previous def. --- src/plotting/colormap_defaults.py | 6 +++--- workflow/Snakefile | 4 ++-- workflow/scripts/verif_single_init.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index a9cd0b3d..06790652 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -54,7 +54,7 @@ def _fallback(): # RMSE: "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, - "WS_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "SP_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, @@ -66,7 +66,7 @@ def _fallback(): # MAE: "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, - "WS_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "SP_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, @@ -79,7 +79,7 @@ def _fallback(): # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, - "WS_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, + "SP_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, diff --git a/workflow/Snakefile b/workflow/Snakefile index a2989eb5..aa1ab24a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -57,7 +57,7 @@ rule experiment_all: run_id=collect_all_candidates(), leadtime=[6], metric=["BIAS"], - param=["WS_10M"], + param=["SP_10M"], region=["centraleurope", "switzerland"], season=["all", "DJF", "MAM", "JJA", "SON"], experiment=EXPERIMENT_HASH @@ -67,7 +67,7 @@ rule experiment_all: baseline_id=collect_all_baselines(), leadtime=[6], metric=["BIAS"], - param=["WS_10M"], + param=["SP_10M"], region=["centraleurope", "switzerland"], season=["all", "DJF", "MAM", "JJA", "SON"], experiment=EXPERIMENT_HASH diff --git a/workflow/scripts/verif_single_init.py b/workflow/scripts/verif_single_init.py index 27b9cbf6..dae6020a 100644 --- a/workflow/scripts/verif_single_init.py +++ b/workflow/scripts/verif_single_init.py @@ -114,10 +114,10 @@ def main(args: ScriptConfig): # before verifying, calculate wind speed: for ds in [fcst, analysis]: if "U_10M" in ds and "V_10M" in ds: - LOG.info("Calculating Wind Speed (WS_10M)...") - ds["WS_10M"] = (ds["U_10M"]**2 + ds["V_10M"]**2)**0.5 + LOG.info("Calculating Wind Speed (SP_10M)...") + ds["SP_10M"] = (ds["U_10M"]**2 + ds["V_10M"]**2)**0.5 # Optional: Add metadata for the netCDF output - ds["WS_10M"].attrs = {"units": "m/s", "long_name": "10m Wind Speed"} + ds["SP_10M"].attrs = {"units": "m/s", "long_name": "10m Wind Speed"} # compute metrics and statistics results = verify(fcst, analysis, args.label, args.analysis_label, args.regions) From 9628fae72db3f287faa9ce3b5bb06ee37fa0a4f7 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 24 Feb 2026 10:09:27 +0100 Subject: [PATCH 051/129] Verification files not temporary any more Some SLURM optimizations. --- workflow/rules/verif.smk | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index 0566ccf2..4d60045d 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -27,13 +27,14 @@ rule verif_metrics_baseline: analysis_label=config["analysis"].get("label"), regions=REGION_TXT, output: - temp(OUT_ROOT / "data/baselines/{baseline_id}/{init_time}/verif.nc"), + OUT_ROOT / "data/baselines/{baseline_id}/{init_time}/verif.nc", log: OUT_ROOT / "logs/verif_metrics_baseline/{baseline_id}-{init_time}.log", resources: cpus_per_task=24, mem_mb=50_000, runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -63,7 +64,7 @@ rule verif_metrics: inference_okfile=rules.execute_inference.output.okfile, analysis_zarr=config["analysis"].get("analysis_zarr"), output: - temp(OUT_ROOT / "data/runs/{run_id}/{init_time}/verif.nc"), + OUT_ROOT / "data/runs/{run_id}/{init_time}/verif.nc", # wildcard_constraints: # run_id="^" # to avoid ambiguitiy with run_baseline_verif # TODO: implement logic to use experiment name instead of run_id as wildcard @@ -81,6 +82,7 @@ rule verif_metrics: cpus_per_task=24, mem_mb=50_000, runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -117,7 +119,8 @@ rule verif_metrics_aggregation: resources: cpus_per_task=24, mem_mb=250_000, - runtime="2h", + runtime="24h", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} {input.verif_nc} --output {output} > {log} 2>&1 From 59d4b12a1369d33ed764550e242ad6f78d7f2464 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 26 Feb 2026 13:53:10 +0100 Subject: [PATCH 052/129] Precipitation plotting: meters to millimeters. --- src/plotting/colormap_defaults.py | 6 +- workflow/scripts/plot_summary_stat_maps.mo.py | 78 ++++++++++++++++++- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 06790652..4fc652e4 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -59,7 +59,7 @@ def _fallback(): "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, - "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.001, 0.0015, 0.002, 0.003, 0.004]} | {"units": "mm"}, + "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, # would ideally want a 6th colour on the high end of the colour scale, but somehow # matplotlib does not do that -> ? @@ -71,7 +71,7 @@ def _fallback(): "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, - "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.001, 0.0015, 0.002, 0.003, 0.004]} | {"units": "mm"}, + "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, # the levels for precip are a bit on the bright side, but still worth keeping consistent with RMSE. # Bias: @@ -84,7 +84,7 @@ def _fallback(): "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, - "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 9), "levels": [-0.001, -0.0005, -0.00025, -0.0001, 0.0001, 0.00025, 0.0005, 0.001]} | {"units": "mm"} + "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 9), "levels": [-1, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 1]} | {"units": "mm"} } CMAP_DEFAULTS = defaultdict(_fallback, _CMAP_DEFAULTS) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 4ff57749..0db54b3f 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -125,6 +125,76 @@ def get_style(param, metric, units_override=None): } return (get_style,) +# @app.cell +# def _(LOG, np): +# """Preprocess fields with pint-based unit conversion and derived quantities.""" +# try: +# import pint # type: ignore + +# _ureg = pint.UnitRegistry() + +# def _k_to_c(arr): +# # robust conversion with pint, fallback if dtype unsupported +# try: +# return (_ureg.Quantity(arr, _ureg.kelvin).to(_ureg.degC)).magnitude +# except Exception: +# return arr - 273.15 + +# def _ms_to_knots(arr): +# # robust conversion with pint, fallback if dtype unsupported +# try: +# return ( +# _ureg.Quantity(arr, _ureg.meter / _ureg.second).to(_ureg.knot) +# ).magnitude +# except Exception: +# return arr * 1.943844 + +# def _m_to_mm(arr): +# # robust conversion with pint, fallback if dtype unsupported +# try: +# return (_ureg.Quantity(arr, _ureg.meter).to(_ureg.millimeter)).magnitude +# except Exception: +# return arr * 1000 + +# except Exception: +# LOG.warning("pint not available; falling back hardcoded conversions") + +# def _k_to_c(arr): +# return arr - 273.15 + +# def _ms_to_knots(arr): +# return arr * 1.943844 + +# def _m_to_mm(arr): +# return arr * 1000 + +# def preprocess_field(param: str, state: dict): +# """ +# - Temperatures: K -> °C +# - Wind speed: sqrt(u^2 + v^2) +# - Precipitation: m -> mm +# Returns: (field_array, units_override or None) +# """ +# fields = state["fields"] +# # temperature variables +# if param in ("T_2M", "TD_2M", "T", "TD"): +# return _k_to_c(fields[param]), "°C" +# # 10m wind speed (allow legacy 'uv' alias) +# if param == "SP_10M": +# u = fields["U_10M"] +# v = fields["V_10M"] +# return np.sqrt(u**2 + v**2), "m/s" +# # wind speed from standard-level components +# if param == "SP": +# u = fields["U"] +# v = fields["V"] +# return np.sqrt(u**2 + v**2), "m/s" +# if param == "TOT_PREC": +# return _m_to_mm(fields[param]), "mm" +# # default: passthrough +# return fields[param], None + +# return (preprocess_field,) @app.cell def _( @@ -162,7 +232,13 @@ def _( # # preprocess field (unit conversion, derived quantities) # field, units_override = preprocess_field(param, state) - plotter.plot_field(subplot, ds.values.ravel(), **get_style(var, metric)) + # Quick fix for precipitation (might have to use preprocess_field in the end (see above)) + if param == "TOT_PREC": + plot_vals = ds.values.ravel()*1000 + else: + plot_vals = ds.values.ravel() + + plotter.plot_field(subplot, plot_vals, **get_style(var, metric)) # subplot.ax.add_geometries( # state["lam_envelope"], # edgecolor="black", From 28ad4c5c343014109b4bd546e1d6ceeb04e59de4 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 26 Feb 2026 13:54:31 +0100 Subject: [PATCH 053/129] Map plotting not on default busy nodes. Change back (clean up in the end) --- workflow/rules/plot.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 10f85c47..23b049d7 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -82,6 +82,7 @@ rule plot_summary_stat_maps: slurm_partition="postproc", cpus_per_task=1, runtime="10m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" params: nc_out_dir=lambda wc: ( Path(OUT_ROOT) / f"data/runs/{wc.run_id}/verif_aggregated.nc" From db60db5feded23a055c7257806c7fde374c1ce4b Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 26 Feb 2026 15:19:20 +0100 Subject: [PATCH 054/129] Preliminary Colour Levels for Wind Bias. Might have to slightly adjust for ICON (emulator). --- src/plotting/colormap_defaults.py | 10 +++++----- workflow/scripts/plot_summary_stat_maps.mo.py | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 4fc652e4..b46fde95 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -77,11 +77,11 @@ def _fallback(): # Bias: # diverging colour scheme for the Bias to reflect the nature of the data (can be positive or negative, symmetric). # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. - "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, - "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, - "SP_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "m/s"}, - "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, - "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels" : np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, + "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "SP_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, + "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 9), "levels": [-1, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 1]} | {"units": "mm"} diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 0db54b3f..728f49db 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -233,6 +233,8 @@ def _( # field, units_override = preprocess_field(param, state) # Quick fix for precipitation (might have to use preprocess_field in the end (see above)) + # for wind speed, preprocess_field only has conversion from m/s to knots + # (not the other way around), so I assume the values are in m/s if param == "TOT_PREC": plot_vals = ds.values.ravel()*1000 else: From 2fcfd8a339050a69f02d93c5dc44cdc729064bbb Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 26 Feb 2026 15:56:52 +0100 Subject: [PATCH 055/129] Replace EXPERIMENT_HASH by EXPERIMENT_NAME Solves a Problem that occurred due to the merge. --- workflow/Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 695abc77..2c11ecb9 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -134,7 +134,7 @@ rule experiment_all: param=["SP_10M"], region=["centraleurope", "switzerland"], season=["all", "DJF", "MAM", "JJA", "SON"], - experiment=EXPERIMENT_HASH + experiment=EXPERIMENT_NAME ), expand( OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", @@ -144,7 +144,7 @@ rule experiment_all: param=["SP_10M"], region=["centraleurope", "switzerland"], season=["all", "DJF", "MAM", "JJA", "SON"], - experiment=EXPERIMENT_HASH + experiment=EXPERIMENT_NAME ) From c00c5a074444fa29201090ddeb615c0b51b16491 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 2 Mar 2026 11:34:47 +0100 Subject: [PATCH 056/129] Colour Levels for Pressure Bias. --- src/plotting/colormap_defaults.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index b46fde95..21bde983 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -82,8 +82,8 @@ def _fallback(): "SP_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, - "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, - "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11)} | {"units": "Pa"}, + "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, + "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 9), "levels": [-1, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 1]} | {"units": "mm"} } From 23d8c9c55a8bf4d68dc0057b25050bbc0a11f399 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 9 Mar 2026 16:56:31 +0100 Subject: [PATCH 057/129] Colour Levels for Pressure MAE and RMSE --- src/plotting/colormap_defaults.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 21bde983..a555d1aa 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -57,8 +57,8 @@ def _fallback(): "SP_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, - "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, + "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, # would ideally want a 6th colour on the high end of the colour scale, but somehow # matplotlib does not do that -> ? @@ -69,17 +69,17 @@ def _fallback(): "SP_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, - "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "Pa"}, + "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, # the levels for precip are a bit on the bright side, but still worth keeping consistent with RMSE. # Bias: # diverging colour scheme for the Bias to reflect the nature of the data (can be positive or negative, symmetric). # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. - "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, - "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, - "SP_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "SP_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, From 84edd34a6c4a994ecd123ca07881655fc53f8b49 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 10 Mar 2026 17:18:26 +0100 Subject: [PATCH 058/129] Colour Levels for 10m Wind MAE and RMSE --- src/plotting/colormap_defaults.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index a555d1aa..7ad4878a 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -52,9 +52,9 @@ def _fallback(): # always start at 0 so that the saturation of the colour corresponds to the error magnitude. # RMSE: - "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, - "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, - "SP_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "SP_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, @@ -64,9 +64,9 @@ def _fallback(): # matplotlib does not do that -> ? # MAE: - "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, - "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, - "SP_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "vmin": 0} | {"units": "m/s"}, + "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "SP_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, From ab36af25739a221cd075b180ab50484044e4e42a Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 20 Mar 2026 12:57:14 +0100 Subject: [PATCH 059/129] Bug fix --- src/verification/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index bded3022..87cb33b3 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -78,8 +78,7 @@ def _compute_scores( dim: list[str], prefix="", suffix="", - source="", - dim=["x", "y"], + source="" ) -> xr.Dataset: """ Compute basic verification metrics between two xarray DataArrays (fcst and obs). From 7dc82dc3a5aad0b1941c531c5737f6decc360466 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 20 Mar 2026 13:46:45 +0100 Subject: [PATCH 060/129] Another Bug Fix. --- src/verification/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index 87cb33b3..2d2fb601 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -115,7 +115,6 @@ def _compute_statistics( prefix="", suffix="", source="", - dim=["x", "y"], ) -> xr.Dataset: """ Compute basic statistics of a xarray DataArray (data). From d33f776bdc0ea9be71c8efface1f5d828f435ff2 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 20 Mar 2026 18:33:50 +0100 Subject: [PATCH 061/129] More Bug fixes. --- workflow/scripts/plot_summary_stat_maps.mo.py | 4 ++-- workflow/scripts/verif_single_init.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 728f49db..48b6cb33 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -215,8 +215,8 @@ def _( import matplotlib.pyplot as plt plotter = StatePlotter( - ds["longitude"].values.ravel(), - ds["latitude"].values.ravel(), + ds["lon"].values.ravel(), + ds["lat"].values.ravel(), outfn.parent, ) fig = plotter.init_geoaxes( diff --git a/workflow/scripts/verif_single_init.py b/workflow/scripts/verif_single_init.py index f0ecdbd5..6b64c85b 100644 --- a/workflow/scripts/verif_single_init.py +++ b/workflow/scripts/verif_single_init.py @@ -72,7 +72,7 @@ def main(args: ScriptConfig): truth = truth.sel(time=fcst.time) # before verifying, calculate wind speed: - for ds in [fcst, analysis]: + for ds in [fcst, truth]: if "U_10M" in ds and "V_10M" in ds: LOG.info("Calculating Wind Speed (SP_10M)...") ds["SP_10M"] = (ds["U_10M"]**2 + ds["V_10M"]**2)**0.5 From 8aa43f7ee43f65cccc988449a9ce8235103d01d4 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 30 Mar 2026 17:57:55 +0200 Subject: [PATCH 062/129] Separate spatial verification files The verification files (verif.nc) do now not contain spatial information any more. Instead, this information is read from the forecast and truth data (grib / zarr), aggregated over forecasts and written into a verif_spatial directory for every run. This has the following advantages: - The spatial information (redundant because it already pre-exists in .grib and .zarr files) is now not unnecessarily written to disk. - The spatial verification pipeline is now separated to some degree from the normal one, meaning that the normal one could be run without the heavier spatial one. - aggregation over forecasts now needs to be performed only for the parameters and lead times for which it is actually requested, instead of for all. This speeds up computations when only a specific subset of maps is of interest. Co-authored-by: Francesco Zanetta --- src/verification/__init__.py | 9 +- workflow/Snakefile | 30 +- workflow/rules/plot.smk | 14 +- workflow/rules/verif.smk | 37 ++ workflow/scripts/plot_summary_stat_maps.mo.py | 10 +- workflow/scripts/verif_spatial.py | 363 ++++++++++++++++++ 6 files changed, 425 insertions(+), 38 deletions(-) create mode 100644 workflow/scripts/verif_spatial.py diff --git a/src/verification/__init__.py b/src/verification/__init__.py index 2d2fb601..2a1f0c37 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -226,16 +226,9 @@ def verify( score = xr.concat(score, dim="region") fcst_statistics = xr.concat(fcst_statistics, dim="region") obs_statistics = xr.concat(obs_statistics, dim="region") - score_spatial = _compute_scores( - fcst_aligned[param], - obs_aligned[param], - prefix=param + ".", - suffix=".spatial", - dim=[], - ) statistics.append(xr.concat([fcst_statistics, obs_statistics], dim="source")) scores.append( - xr.merge([score, score_spatial], join="outer", compat="no_conflicts") + xr.merge([score], join="outer", compat="no_conflicts") ) scores = _merge_metrics(scores) diff --git a/workflow/Snakefile b/workflow/Snakefile index 2c11ecb9..edd220ce 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -127,25 +127,25 @@ rule experiment_all: experiment=EXPERIMENT_NAME, ), expand( - OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + rules.plot_summary_stat_maps.output, run_id=collect_all_candidates(), leadtime=[6], metric=["BIAS"], param=["SP_10M"], - region=["centraleurope", "switzerland"], - season=["all", "DJF", "MAM", "JJA", "SON"], - experiment=EXPERIMENT_NAME - ), - expand( - OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", - baseline_id=collect_all_baselines(), - leadtime=[6], - metric=["BIAS"], - param=["SP_10M"], - region=["centraleurope", "switzerland"], - season=["all", "DJF", "MAM", "JJA", "SON"], - experiment=EXPERIMENT_NAME - ) + region=["centraleurope", "switzerland"], + season=["all", "DJF", "MAM", "JJA", "SON"], + experiment=EXPERIMENT_NAME, + ), + # expand( + # OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + # baseline_id=collect_all_baselines(), + # leadtime=[6], + # metric=["BIAS"], + # param=["SP_10M"], + # region=["centraleurope", "switzerland"], + # season=["all", "DJF", "MAM", "JJA", "SON"], + # experiment=EXPERIMENT_NAME + # ) rule showcase_all: diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 14e2e4fc..ffe543b0 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -145,7 +145,7 @@ rule plot_summary_stat_maps: localrule: True input: script="workflow/scripts/plot_summary_stat_maps.mo.py", - verif_file=OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", + verif_file=OUT_ROOT / "data/runs/{run_id}/verif_spatial/{param}_{leadtime}.nc", output: OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", wildcard_constraints: @@ -155,24 +155,18 @@ rule plot_summary_stat_maps: cpus_per_task=1, runtime="10m", slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" - params: - nc_out_dir=lambda wc: ( - Path(OUT_ROOT) / f"data/runs/{wc.run_id}/verif_aggregated.nc" - # not sure how to do this, because the baselines are in, e.g., output/data/baselines/COSMO-E/verif_aggregated.nc - # and the runs are in output/data/runs/runID/verif_aggregated.nc - ).resolve(), shell: """ export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) - python {input.script} \ + uv run python {input.script} \ --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ - --season {wildcards.season} \ + --season {wildcards.season} # interactive editing (needs to set localrule: True and use only one core) # marimo edit {input.script} -- \ # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ - # --season {wildcards.season} \ + # --season {wildcards.season} """ use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index 03ca095a..13a2970f 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -96,6 +96,43 @@ rule verif_metrics: --output {output} > {log} 2>&1 """ +rule verif_metrics_spatial: + input: + "src/verification/__init__.py", + "src/data_input/__init__.py", + script="workflow/scripts/verif_spatial.py", + inference_okfiles=lambda wc: expand( + rules.execute_inference.output.okfile, + init_time=_restrict_reftimes_to_hours(REFTIMES), + allow_missing=True, + ), + truth=config["truth"]["root"], + output: + OUT_ROOT / "data/runs/{run_id}/verif_spatial/{param}_{leadtime}.nc", + # wildcard_constraints: + # run_id="^" # to avoid ambiguitiy with run_baseline_verif + # TODO: implement logic to use experiment name instead of run_id as wildcard + params: + fcst_label=lambda wc: RUN_CONFIGS[wc.run_id].get("label"), + fcst_steps=lambda wc: RUN_CONFIGS[wc.run_id]["steps"], + truth_label=config["truth"]["label"], + log: + OUT_ROOT / "logs/verif_metrics_spatial/{run_id}-{param}-{leadtime}.log", + resources: + cpus_per_task=24, + mem_mb=50_000, + runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + shell: + """ + uv run workflow/scripts/verif_spatial.py \ + --run_root output/data/runs/{wildcards.run_id} \ + --truth {input.truth} \ + --step {wildcards.leadtime} \ + --param {wildcards.param} \ + --output {output} > {log} 2>&1 + """ + def _restrict_reftimes_to_hours(reftimes, hours=None): """Restrict the reference times to specific hours.""" diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 48b6cb33..fa4adc82 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -92,10 +92,10 @@ def _(ArgumentParser, Path, np): @app.cell -def _(init_hour, lead_time, metric, param, season, verif_file, xr): +def _(metric, param, verif_file, xr): ds = xr.open_dataset(verif_file) - var = f"{param}.{metric}.spatial" - ds = ds[var].sel(init_hour=init_hour, lead_time=lead_time, season=season) + var = f"{param}.{metric}" + ds = ds[var] ds return ds, var @@ -107,7 +107,7 @@ def get_style(param, metric, units_override=None): still needs to be passed as arguments to tripcolor()/tricontourf(). """ metric_key = f"{param}.{metric}.spatial" - cfg = CMAP_DEFAULTS[metric_key] if metric_key in CMAP_DEFAULTS else CMAP_DEFAULTS[param] + cfg = CMAP_DEFAULTS[metric_key] if metric_key in CMAP_DEFAULTS else CMAP_DEFAULTS.get(param, {}) units = units_override if units_override is not None else cfg.get("units", "") return { "style": ekp.styles.Style( @@ -240,7 +240,7 @@ def _( else: plot_vals = ds.values.ravel() - plotter.plot_field(subplot, plot_vals, **get_style(var, metric)) + plotter.plot_field(subplot, plot_vals, **get_style(param, metric)) # subplot.ax.add_geometries( # state["lam_envelope"], # edgecolor="black", diff --git a/workflow/scripts/verif_spatial.py b/workflow/scripts/verif_spatial.py new file mode 100644 index 00000000..58cb2624 --- /dev/null +++ b/workflow/scripts/verif_spatial.py @@ -0,0 +1,363 @@ +"""Compute spatial maps of temporally-aggregated forecast errors. + +For a fixed lead time and variable, iterates over all initialisation times found +under a run directory, loads the corresponding GRIB forecast field and the +matching truth slice from a reference zarr, maps the forecast onto the truth +grid, and accumulates running error statistics without ever holding the full +time series in memory. The final BIAS / RMSE / MAE maps are written to a +NetCDF file. + +Usage +----- + uv run workflow/scripts/verif_spatial.py \\ + output/data/runs/ \\ + --truth /path/to/truth.zarr \\ + --step 24 \\ + --param T_2M +""" + +import logging +from argparse import ArgumentParser, Namespace +from datetime import datetime, timedelta +from pathlib import Path + +import numpy as np +import xarray as xr + +from data_input import load_fct_data_from_grib +from verification.spatial import map_forecast_to_truth + +LOG = logging.getLogger(__name__) +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) + +DATETIME_FMT = "%Y%m%d%H%M" + +# Maps from standard parameter names to zarr variable names. +# COSMO-2e zarrs use short CF names; COSMO-1e zarrs keep the COSMO names. +_PARAMS_MAP_CO2 = { + "T_2M": "2t", + "TD_2M": "2d", + "U_10M": "10u", + "V_10M": "10v", + "PS": "sp", + "PMSL": "msl", + "TOT_PREC": "tp", +} +_PARAMS_MAP_CO1 = {k: k.replace("TOT_PREC", "TOT_PREC_6H") for k in _PARAMS_MAP_CO2} + +# Derived variables and the components they require. +_DERIVED = { + "SP_10M": ("U_10M", "V_10M"), +} + + +def _params_map(truth_root: Path) -> dict[str, str]: + return _PARAMS_MAP_CO2 if "co2" in truth_root.name else _PARAMS_MAP_CO1 + + +def _compute_derived(ds: xr.Dataset, param: str) -> xr.DataArray: + """Compute a derived variable from its components already present in *ds*.""" + if param == "SP_10M": + return (ds["U_10M"] ** 2 + ds["V_10M"] ** 2) ** 0.5 + raise ValueError(f"No recipe for derived variable '{param}'") + + +# --------------------------------------------------------------------------- +# Truth loading +# --------------------------------------------------------------------------- + +def _open_zarr_component(root: Path, param: str) -> xr.DataArray: + """Open a single native zarr variable lazily as a DataArray.""" + zarr_param = _params_map(root)[param] + + ds = xr.open_zarr(root, consolidated=False) + ds = ds.set_index(time="dates") + + # Extract lat/lon before selecting on variable (they live on cell only). + spatial_dim = "cell" + lat = ds["latitudes"] if "latitudes" in ds else None + lon = ds["longitudes"] if "longitudes" in ds else None + + ds = ds.assign_coords(variable=ds.attrs["variables"]) + ds = ds.sel(variable=zarr_param).squeeze("ensemble", drop=True) + + # Recover 2-D spatial shape when stored as a flat cell dimension. + if len(ds.attrs["field_shape"]) == 2: + ny, nx = ds.attrs["field_shape"] + y_idx, x_idx = np.unravel_index(np.arange(ny * nx), (ny, nx)) + ds = ds.assign_coords(y=(spatial_dim, y_idx), x=(spatial_dim, x_idx)) + ds = ds.set_index(**{spatial_dim: ("y", "x")}).unstack(spatial_dim) + spatial_dim = None # now (y, x) + + da = ds["data"].rename(param).drop_vars("variable", errors="ignore") + + # Attach lat/lon as coordinates on the spatial dimension(s). + if lat is not None and lon is not None: + if spatial_dim is not None: + # flat 1-D case: cell/values dim + da = da.assign_coords(lat=(spatial_dim, lat.values), lon=(spatial_dim, lon.values)) + else: + # 2-D case: lat/lon still on original flat index — attach via unstack + da = da.assign_coords(lat=(["y", "x"], lat.values.reshape(ny, nx)), + lon=(["y", "x"], lon.values.reshape(ny, nx))) + + return da + + +def open_truth_zarr(root: Path, param: str) -> xr.DataArray: + """Open the truth zarr lazily and return a DataArray for *param*. + + For derived variables (e.g. SP_10M) the required components are loaded and + the derivation is applied on the fly. The returned DataArray has dimensions + ``(time, y, x)`` or ``(time, values)`` and always exposes ``lat``/``lon``. + """ + if param in _DERIVED: + components = { + c: _open_zarr_component(root, c).drop_vars("variable", errors="ignore") + for c in _DERIVED[param] + } + ds = xr.Dataset(components) + return _compute_derived(ds, param) + return _open_zarr_component(root, param) + + +# --------------------------------------------------------------------------- +# Init-time discovery +# --------------------------------------------------------------------------- + +def iter_init_dirs(run_root: Path) -> list[tuple[datetime, Path]]: + """Return ``(reftime, grib_dir)`` pairs for every complete init time. + + Expects subdirectories named ``YYYYMMDDHHMI`` directly under *run_root*. + GRIB files may live either directly in the init-time directory or inside a + ``grib/`` subdirectory. + """ + result = [] + for d in sorted(run_root.iterdir()): + if not d.is_dir(): + continue + try: + reftime = datetime.strptime(d.name, DATETIME_FMT) + except ValueError: + continue + grib_dir = d / "grib" if (d / "grib").is_dir() else d + if not any(grib_dir.glob("*.grib")): + LOG.debug("No GRIB files in %s, skipping", grib_dir) + continue + result.append((reftime, grib_dir)) + return result + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(args: Namespace) -> None: + LOG.info("=" * 60) + LOG.info("Spatial verification param=%s step=%dh", args.param, args.step) + LOG.info("Run root : %s", args.run_root) + LOG.info("Truth : %s", args.truth) + LOG.info("Output : %s", args.output) + LOG.info("=" * 60) + + # Open the truth zarr once; individual time slices are loaded on demand. + truth_da = open_truth_zarr(args.truth, args.param) + # Normalise to datetime64[ns] so membership checks work regardless of zarr precision. + truth_da = truth_da.assign_coords( + time=truth_da.time.values.astype("datetime64[ns]") + ) + # Rename flat spatial dim to 'values' if the zarr uses 'cell'. + if "cell" in truth_da.dims: + truth_da = truth_da.rename({"cell": "values"}) + truth_times = set(truth_da.time.values) # keep as datetime64, tolist() yields ints for ns precision + LOG.info("Truth opened lazily: %s", truth_da) + + init_dirs = iter_init_dirs(args.run_root) + LOG.info("Found %d init time directories", len(init_dirs)) + + step_td = timedelta(hours=args.step) + + # Running accumulators – initialised on the first successfully processed + # sample so that we can infer the spatial shape from the data. + accum_n: np.ndarray | None = None + accum_sum_e: np.ndarray | None = None + accum_sum_se: np.ndarray | None = None + accum_sum_ae: np.ndarray | None = None + ref_truth_slice: xr.DataArray | None = None # kept for output coordinates + + n_ok = 0 + n_skip = 0 + + for reftime, grib_dir in init_dirs: + valid_time = np.datetime64(reftime + step_td).astype("datetime64[ns]") + + if valid_time not in truth_times: + LOG.debug("Valid time %s not in truth, skipping %s", valid_time, reftime) + n_skip += 1 + continue + + LOG.info( + "Processing reftime=%s valid=%s", + reftime.strftime(DATETIME_FMT), + valid_time, + ) + + # --- load forecast --- + grib_params = list(_DERIVED[args.param]) if args.param in _DERIVED else [args.param] + try: + fcst = load_fct_data_from_grib( + root=grib_dir, + reftime=reftime, + steps=[args.step], + params=grib_params, + ) + except Exception as exc: + LOG.warning("Could not load GRIB for %s: %s", reftime, exc) + n_skip += 1 + continue + + # Drop lead_time dimension (single step requested). + if "lead_time" in fcst.dims: + fcst = fcst.sel(lead_time=np.timedelta64(args.step, "h")) + + # Compute derived variable if needed. + if args.param in _DERIVED: + fcst = fcst.assign({args.param: _compute_derived(fcst, args.param)}) + + # --- load truth slice --- + truth_slice = truth_da.sel(time=valid_time).compute() + # For derived variables truth_da is already the derived DataArray, + # so wrap it in a Dataset for map_forecast_to_truth compatibility. + truth_ds = truth_slice.to_dataset(name=args.param) if isinstance(truth_slice, xr.DataArray) else truth_slice + + # --- map forecast onto truth grid --- + try: + fcst_mapped = map_forecast_to_truth(fcst, truth_ds) + except Exception as exc: + LOG.warning("Spatial mapping failed for %s: %s", reftime, exc) + n_skip += 1 + continue + + fcst_param = fcst_mapped[args.param] + # Squeeze any ensemble/eps dimension (deterministic run stored with size-1 eps). + for dim in ["eps", "ensemble", "number"]: + if dim in fcst_param.dims and fcst_param.sizes[dim] == 1: + fcst_param = fcst_param.squeeze(dim, drop=True) + fcst_vals = fcst_param.values + truth_vals = truth_slice.values + error = fcst_vals - truth_vals # shape: spatial dims of truth + + # --- initialise accumulators --- + if accum_n is None: + accum_n = np.zeros(error.shape, dtype=np.int64) + accum_sum_e = np.zeros(error.shape, dtype=np.float64) + accum_sum_se = np.zeros(error.shape, dtype=np.float64) + accum_sum_ae = np.zeros(error.shape, dtype=np.float64) + ref_truth_slice = truth_slice + + # --- accumulate (NaN-safe) --- + valid = ~np.isnan(error) + accum_n[valid] += 1 + accum_sum_e[valid] += error[valid] + accum_sum_se[valid] += error[valid] ** 2 + accum_sum_ae[valid] += np.abs(error[valid]) + n_ok += 1 + + LOG.info("Finished: %d init times processed, %d skipped", n_ok, n_skip) + + if n_ok == 0: + LOG.error("No data could be processed – no output written.") + return + + # --- compute aggregate maps --- + with np.errstate(invalid="ignore", divide="ignore"): + bias = np.where(accum_n > 0, accum_sum_e / accum_n, np.nan).astype(np.float32) + rmse = np.where(accum_n > 0, np.sqrt(accum_sum_se / accum_n), np.nan).astype( + np.float32 + ) + mae = np.where(accum_n > 0, accum_sum_ae / accum_n, np.nan).astype(np.float32) + count = np.where(accum_n > 0, accum_n, np.nan).astype(np.float32) + + # Build a spatial template: keep only spatial dims and lat/lon coords, + # dropping scalar coords like time (added by .sel). + spatial_coords = { + c: ref_truth_slice[c] + for c in ref_truth_slice.coords + if set(ref_truth_slice[c].dims).issubset(set(ref_truth_slice.dims)) + and c != "time" + } + + def _da(data: np.ndarray) -> xr.DataArray: + return xr.DataArray(data, dims=ref_truth_slice.dims, coords=spatial_coords) + + out = xr.Dataset( + { + f"{args.param}.BIAS": _da(bias), + f"{args.param}.RMSE": _da(rmse), + f"{args.param}.MAE": _da(mae), + f"{args.param}.N": _da(count), + }, + attrs={ + "param": args.param, + "step_h": args.step, + "run_root": str(args.run_root), + "n_processed": n_ok, + "n_skipped": n_skip, + }, + ) + + LOG.info("Output dataset:\n%s", out) + args.output.parent.mkdir(parents=True, exist_ok=True) + out.to_netcdf(args.output) + LOG.info("Saved to %s", args.output) + + +if __name__ == "__main__": + parser = ArgumentParser( + description=( + "Compute spatial maps of temporally-aggregated forecast errors " + "by streaming over GRIB files from a model run." + ) + ) + parser.add_argument( + "--run_root", + type=Path, + help="Root directory of a model run (e.g. output/data/runs/).", + ) + parser.add_argument( + "--truth", + type=Path, + required=True, + help="Path to the reference zarr dataset.", + ) + parser.add_argument( + "--step", + type=int, + required=True, + help="Forecast lead time in hours (e.g. 24).", + ) + parser.add_argument( + "--param", + type=str, + required=True, + help="Variable to verify (e.g. T_2M, TD_2M, U_10M).", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help=( + "Output NetCDF file. " + "Default: /verif_spatial__steph.nc" + ), + ) + args = parser.parse_args() + + if args.output is None: + args.output = ( + args.run_root / f"verif_spatial_{args.param}_step{args.step:03d}h.nc" + ) + + main(args) From c291f0fcfcf5aa410b8db8b590d70004a96ec8d5 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 31 Mar 2026 17:46:41 +0200 Subject: [PATCH 063/129] Introduce Season Handling (not thoroughly tested) --- workflow/scripts/plot_summary_stat_maps.mo.py | 4 +- workflow/scripts/verif_spatial.py | 95 ++++++++++++------- 2 files changed, 62 insertions(+), 37 deletions(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index fa4adc82..27231c64 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -92,10 +92,10 @@ def _(ArgumentParser, Path, np): @app.cell -def _(metric, param, verif_file, xr): +def _(metric, param, season, verif_file, xr): ds = xr.open_dataset(verif_file) var = f"{param}.{metric}" - ds = ds[var] + ds = ds[var].sel(season=season) ds return ds, var diff --git a/workflow/scripts/verif_spatial.py b/workflow/scripts/verif_spatial.py index 58cb2624..246b8c68 100644 --- a/workflow/scripts/verif_spatial.py +++ b/workflow/scripts/verif_spatial.py @@ -34,6 +34,20 @@ DATETIME_FMT = "%Y%m%d%H%M" +SEASONS = ["DJF", "MAM", "JJA", "SON", "all"] + + +def _season_of(dt: datetime) -> str: + """Return the meteorological season string for a given datetime.""" + month = dt.month + if month in (12, 1, 2): + return "DJF" + if month in (3, 4, 5): + return "MAM" + if month in (6, 7, 8): + return "JJA" + return "SON" + # Maps from standard parameter names to zarr variable names. # COSMO-2e zarrs use short CF names; COSMO-1e zarrs keep the COSMO names. _PARAMS_MAP_CO2 = { @@ -179,12 +193,13 @@ def main(args: Namespace) -> None: step_td = timedelta(hours=args.step) - # Running accumulators – initialised on the first successfully processed - # sample so that we can infer the spatial shape from the data. - accum_n: np.ndarray | None = None - accum_sum_e: np.ndarray | None = None - accum_sum_se: np.ndarray | None = None - accum_sum_ae: np.ndarray | None = None + # Running accumulators per season – initialised on the first successfully + # processed sample so that we can infer the spatial shape from the data. + # Each entry is a numpy array over the spatial dimension(s). + accum_n: dict[str, np.ndarray | None] = {s: None for s in SEASONS} + accum_sum_e: dict[str, np.ndarray | None] = {s: None for s in SEASONS} + accum_sum_se: dict[str, np.ndarray | None] = {s: None for s in SEASONS} + accum_sum_ae: dict[str, np.ndarray | None] = {s: None for s in SEASONS} ref_truth_slice: xr.DataArray | None = None # kept for output coordinates n_ok = 0 @@ -249,20 +264,23 @@ def main(args: Namespace) -> None: truth_vals = truth_slice.values error = fcst_vals - truth_vals # shape: spatial dims of truth - # --- initialise accumulators --- - if accum_n is None: - accum_n = np.zeros(error.shape, dtype=np.int64) - accum_sum_e = np.zeros(error.shape, dtype=np.float64) - accum_sum_se = np.zeros(error.shape, dtype=np.float64) - accum_sum_ae = np.zeros(error.shape, dtype=np.float64) + # --- initialise accumulators on first valid sample --- + if accum_n["all"] is None: + for s in SEASONS: + accum_n[s] = np.zeros(error.shape, dtype=np.int64) + accum_sum_e[s] = np.zeros(error.shape, dtype=np.float64) + accum_sum_se[s] = np.zeros(error.shape, dtype=np.float64) + accum_sum_ae[s] = np.zeros(error.shape, dtype=np.float64) ref_truth_slice = truth_slice - # --- accumulate (NaN-safe) --- + # --- accumulate into the matching season bucket and "all" (NaN-safe) --- + season = _season_of(reftime) valid = ~np.isnan(error) - accum_n[valid] += 1 - accum_sum_e[valid] += error[valid] - accum_sum_se[valid] += error[valid] ** 2 - accum_sum_ae[valid] += np.abs(error[valid]) + for s in [season, "all"]: + accum_n[s][valid] += 1 + accum_sum_e[s][valid] += error[valid] + accum_sum_se[s][valid] += error[valid] ** 2 + accum_sum_ae[s][valid] += np.abs(error[valid]) n_ok += 1 LOG.info("Finished: %d init times processed, %d skipped", n_ok, n_skip) @@ -271,33 +289,40 @@ def main(args: Namespace) -> None: LOG.error("No data could be processed – no output written.") return - # --- compute aggregate maps --- - with np.errstate(invalid="ignore", divide="ignore"): - bias = np.where(accum_n > 0, accum_sum_e / accum_n, np.nan).astype(np.float32) - rmse = np.where(accum_n > 0, np.sqrt(accum_sum_se / accum_n), np.nan).astype( - np.float32 - ) - mae = np.where(accum_n > 0, accum_sum_ae / accum_n, np.nan).astype(np.float32) - count = np.where(accum_n > 0, accum_n, np.nan).astype(np.float32) - - # Build a spatial template: keep only spatial dims and lat/lon coords, - # dropping scalar coords like time (added by .sel). + # --- compute aggregate maps per season, then stack into a season dimension --- spatial_coords = { c: ref_truth_slice[c] for c in ref_truth_slice.coords if set(ref_truth_slice[c].dims).issubset(set(ref_truth_slice.dims)) and c != "time" } - - def _da(data: np.ndarray) -> xr.DataArray: - return xr.DataArray(data, dims=ref_truth_slice.dims, coords=spatial_coords) + spatial_dims = list(ref_truth_slice.dims) + out_dims = ["season"] + spatial_dims + out_coords = {"season": SEASONS, **spatial_coords} + + def _seasonal_da(compute_fn) -> xr.DataArray: + """Stack per-season arrays into a (season, *spatial) DataArray.""" + slices = [] + for s in SEASONS: + n = accum_n[s] + with np.errstate(invalid="ignore", divide="ignore"): + slices.append(compute_fn(n, s).astype(np.float32)) + return xr.DataArray(np.stack(slices), dims=out_dims, coords=out_coords) out = xr.Dataset( { - f"{args.param}.BIAS": _da(bias), - f"{args.param}.RMSE": _da(rmse), - f"{args.param}.MAE": _da(mae), - f"{args.param}.N": _da(count), + f"{args.param}.BIAS": _seasonal_da( + lambda n, s: np.where(n > 0, accum_sum_e[s] / n, np.nan) + ), + f"{args.param}.RMSE": _seasonal_da( + lambda n, s: np.where(n > 0, np.sqrt(accum_sum_se[s] / n), np.nan) + ), + f"{args.param}.MAE": _seasonal_da( + lambda n, s: np.where(n > 0, accum_sum_ae[s] / n, np.nan) + ), + f"{args.param}.N": _seasonal_da( + lambda n, s: np.where(n > 0, n, np.nan) + ), }, attrs={ "param": args.param, From 8afb9ac55df24c2133d4ca60519246a26380071e Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 1 Apr 2026 13:56:12 +0200 Subject: [PATCH 064/129] Additional Log Statements for Map Plotting plus handling of the special case when a map is all-NaN (then a uniform grey map is plotted, with an annotation). --- workflow/rules/plot.smk | 6 +++- workflow/scripts/plot_summary_stat_maps.mo.py | 30 +++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index ffe543b0..c8550d4d 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -150,6 +150,8 @@ rule plot_summary_stat_maps: OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits + log: + OUT_ROOT / "logs/plot_summary_stat_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", resources: slurm_partition="postproc", cpus_per_task=1, @@ -161,7 +163,7 @@ rule plot_summary_stat_maps: uv run python {input.script} \ --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ - --season {wildcards.season} + --season {wildcards.season} > {log} 2>&1 # interactive editing (needs to set localrule: True and use only one core) # marimo edit {input.script} -- \ # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ @@ -175,6 +177,8 @@ use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", output: OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + log: + OUT_ROOT / "logs/plot_summary_stat_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", params: nc_out_dir=lambda wc: ( Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 27231c64..50d2d414 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -92,10 +92,14 @@ def _(ArgumentParser, Path, np): @app.cell -def _(metric, param, season, verif_file, xr): +def _(LOG, metric, param, season, verif_file, xr): ds = xr.open_dataset(verif_file) + LOG.info("Opened dataset: %s", ds) var = f"{param}.{metric}" + LOG.info("Selecting variable '%s' for season '%s'", var, season) ds = ds[var].sel(season=season) + LOG.info("Selected DataArray: dims=%s, shape=%s, dtype=%s", ds.dims, ds.shape, ds.dtype) + LOG.info("Value range: min=%.4g, max=%.4g, n_nan=%d", float(ds.min()), float(ds.max()), int(ds.isnull().sum())) ds return ds, var @@ -205,10 +209,11 @@ def _( get_style, lead_time, metric, + np, outfn, param, - region, - season, + region, + season, var, ): # plot individual fields @@ -233,14 +238,27 @@ def _( # field, units_override = preprocess_field(param, state) # Quick fix for precipitation (might have to use preprocess_field in the end (see above)) - # for wind speed, preprocess_field only has conversion from m/s to knots + # for wind speed, preprocess_field only has conversion from m/s to knots # (not the other way around), so I assume the values are in m/s if param == "TOT_PREC": plot_vals = ds.values.ravel()*1000 else: plot_vals = ds.values.ravel() - - plotter.plot_field(subplot, plot_vals, **get_style(param, metric)) + LOG.info("plot_vals: shape=%s, min=%.4g, max=%.4g, n_nan=%d", + plot_vals.shape, float(np.nanmin(plot_vals)), float(np.nanmax(plot_vals)), int(np.isnan(plot_vals).sum())) + + style_kwargs = get_style(param, metric) + LOG.info("style_kwargs: %s", style_kwargs) + + if np.all(np.isnan(plot_vals)): + LOG.warning("All values are NaN for %s %s season=%s — plotting empty map.", param, metric, season) + import matplotlib.patches as mpatches + subplot.ax.set_facecolor("#cccccc") + subplot.standard_layers() + grey_patch = mpatches.Patch(color="#cccccc", label="No data") + subplot.ax.legend(handles=[grey_patch], loc="lower left", fontsize=8) + else: + plotter.plot_field(subplot, plot_vals, **style_kwargs) # subplot.ax.add_geometries( # state["lam_envelope"], # edgecolor="black", From f8a66027e8bcf8159e5feef231efda46d269dd55 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 1 Apr 2026 16:24:09 +0200 Subject: [PATCH 065/129] Fix all-NaN spatial verification output for cumulative params (TOT_PREC) TOT_PREC GRIB values are cumulative totals that are disaggregated via diff(lead_time) inside load_fct_data_from_grib. Loading only the requested step meant the diff had a single element and always returned NaN, producing all-NaN error maps for all seasons. Fix: detect cumulative params via _CUMULATIVE_PARAMS and load the preceding available GRIB step alongside the requested one, so the diff produces a valid result at the target lead time. Also add first-iteration debug logging (fcst/truth/mapped/error shape, range, NaN count) and a per-iteration warning when the error is all-NaN, to make similar issues immediately diagnosable from the logs. --- workflow/scripts/verif_spatial.py | 81 ++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/workflow/scripts/verif_spatial.py b/workflow/scripts/verif_spatial.py index 246b8c68..28e8911d 100644 --- a/workflow/scripts/verif_spatial.py +++ b/workflow/scripts/verif_spatial.py @@ -137,6 +137,27 @@ def open_truth_zarr(root: Path, param: str) -> xr.DataArray: return _open_zarr_component(root, param) +# --------------------------------------------------------------------------- +# GRIB step helpers +# --------------------------------------------------------------------------- + +# Parameters whose GRIB values are cumulative totals and must be disaggregated +# via diff before verification. For these, the preceding step must also be +# loaded so that load_fct_data_from_grib's diff produces a valid result. +_CUMULATIVE_PARAMS = {"TOT_PREC"} + + +def _preceding_step(grib_dir: Path, step: int) -> int | None: + """Return the largest available step smaller than *step* in *grib_dir*.""" + available = sorted( + int(f.stem.split("_")[-1]) + for f in grib_dir.glob("*.grib") + if f.stem.split("_")[-1].isdigit() + ) + smaller = [s for s in available if s < step] + return smaller[-1] if smaller else None + + # --------------------------------------------------------------------------- # Init-time discovery # --------------------------------------------------------------------------- @@ -219,13 +240,30 @@ def main(args: Namespace) -> None: valid_time, ) + first_iter = n_ok == 0 + # --- load forecast --- grib_params = list(_DERIVED[args.param]) if args.param in _DERIVED else [args.param] + + # Cumulative params (e.g. TOT_PREC) are disaggregated via diff inside + # load_fct_data_from_grib, so we need to also load the preceding step + # to get a valid result at the requested step. + if args.param in _CUMULATIVE_PARAMS: + prev_step = _preceding_step(grib_dir, args.step) + if prev_step is None: + LOG.warning("No preceding step found for cumulative param %s at step %d, skipping %s", + args.param, args.step, reftime) + n_skip += 1 + continue + load_steps = [prev_step, args.step] + else: + load_steps = [args.step] + try: fcst = load_fct_data_from_grib( root=grib_dir, reftime=reftime, - steps=[args.step], + steps=load_steps, params=grib_params, ) except Exception as exc: @@ -233,7 +271,7 @@ def main(args: Namespace) -> None: n_skip += 1 continue - # Drop lead_time dimension (single step requested). + # Drop lead_time dimension (select only the requested step). if "lead_time" in fcst.dims: fcst = fcst.sel(lead_time=np.timedelta64(args.step, "h")) @@ -241,12 +279,32 @@ def main(args: Namespace) -> None: if args.param in _DERIVED: fcst = fcst.assign({args.param: _compute_derived(fcst, args.param)}) + if first_iter: + LOG.info("fcst (after step selection): %s", fcst) + fcst_raw = fcst[args.param].values if args.param in fcst else None + if fcst_raw is not None: + n_nan_fcst = int(np.isnan(fcst_raw).sum()) + LOG.info("fcst[%s]: shape=%s, min=%.4g, max=%.4g, n_nan=%d", + args.param, fcst_raw.shape, + float(np.nanmin(fcst_raw)) if n_nan_fcst < fcst_raw.size else float("nan"), + float(np.nanmax(fcst_raw)) if n_nan_fcst < fcst_raw.size else float("nan"), + n_nan_fcst) + # --- load truth slice --- truth_slice = truth_da.sel(time=valid_time).compute() # For derived variables truth_da is already the derived DataArray, # so wrap it in a Dataset for map_forecast_to_truth compatibility. truth_ds = truth_slice.to_dataset(name=args.param) if isinstance(truth_slice, xr.DataArray) else truth_slice + if first_iter: + truth_raw = truth_slice.values + n_nan_truth = int(np.isnan(truth_raw).sum()) + LOG.info("truth_slice[%s]: shape=%s, min=%.4g, max=%.4g, n_nan=%d", + args.param, truth_raw.shape, + float(np.nanmin(truth_raw)) if n_nan_truth < truth_raw.size else float("nan"), + float(np.nanmax(truth_raw)) if n_nan_truth < truth_raw.size else float("nan"), + n_nan_truth) + # --- map forecast onto truth grid --- try: fcst_mapped = map_forecast_to_truth(fcst, truth_ds) @@ -264,6 +322,25 @@ def main(args: Namespace) -> None: truth_vals = truth_slice.values error = fcst_vals - truth_vals # shape: spatial dims of truth + if first_iter: + n_nan_mapped = int(np.isnan(fcst_vals).sum()) + LOG.info("fcst_mapped[%s]: shape=%s, min=%.4g, max=%.4g, n_nan=%d", + args.param, fcst_vals.shape, + float(np.nanmin(fcst_vals)) if n_nan_mapped < fcst_vals.size else float("nan"), + float(np.nanmax(fcst_vals)) if n_nan_mapped < fcst_vals.size else float("nan"), + n_nan_mapped) + n_nan_err = int(np.isnan(error).sum()) + LOG.info("error: shape=%s, min=%.4g, max=%.4g, n_nan=%d / %d", + error.shape, + float(np.nanmin(error)) if n_nan_err < error.size else float("nan"), + float(np.nanmax(error)) if n_nan_err < error.size else float("nan"), + n_nan_err, error.size) + + n_nan_error = int(np.isnan(error).sum()) + if n_nan_error == error.size: + LOG.warning("reftime=%s: error is all-NaN (%d points) — nothing accumulated.", + reftime.strftime(DATETIME_FMT), error.size) + # --- initialise accumulators on first valid sample --- if accum_n["all"] is None: for s in SEASONS: From 26f6f9d9e2be2f15be923d115af866ceb6d2bcb7 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 1 Apr 2026 16:33:16 +0200 Subject: [PATCH 066/129] Add opt-in spatial verification via `evalml experiment --spatial` Introduce a dedicated `spatial_all` Snakemake target and a new `--spatial` flag on `evalml experiment`. Spatial verification (aggregation + map plots) is now only triggered on explicit request, keeping the default experiment run lean. Spatial parameters (params, leadtimes, metrics, regions, seasons) are configured via an optional `spatial_verification:` block in the config YAML, backed by a new `SpatialVerificationConfig` Pydantic model with sensible defaults. All existing configs have been updated with the block for easy customisation. --- config/forecasters-co1e.yaml | 22 ++++++++++++++++ config/forecasters-co2-disentangled.yaml | 22 ++++++++++++++++ config/forecasters-co2-mod-01.yaml | 22 ++++++++++++++++ config/forecasters-co2.yaml | 22 ++++++++++++++++ config/forecasters-ich1-oper-fixed.yaml | 26 +++++++++++++++++-- config/forecasters-ich1-oper.yaml | 22 ++++++++++++++++ config/forecasters-ich1.yaml | 22 ++++++++++++++++ config/forecasters-ich1_mod_01.yaml | 22 ++++++++++++++++ config/forecasters-ich1_mod_02_1yr.yaml | 22 ++++++++++++++++ config/interpolators-co2.yaml | 22 ++++++++++++++++ src/evalml/cli.py | 12 +++++++-- src/evalml/config.py | 32 ++++++++++++++++++++++++ workflow/Snakefile | 25 ++++++++---------- 13 files changed, 274 insertions(+), 19 deletions(-) diff --git a/config/forecasters-co1e.yaml b/config/forecasters-co1e.yaml index 83846ec4..1a1bac39 100644 --- a/config/forecasters-co1e.yaml +++ b/config/forecasters-co1e.yaml @@ -57,3 +57,25 @@ profile: jobs: 50 batch_rules: plot_forecast_frame: 32 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/config/forecasters-co2-disentangled.yaml b/config/forecasters-co2-disentangled.yaml index c9a75f2c..e962767a 100644 --- a/config/forecasters-co2-disentangled.yaml +++ b/config/forecasters-co2-disentangled.yaml @@ -76,3 +76,25 @@ profile: runtime: "1h" gpus: 0 jobs: 50 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/config/forecasters-co2-mod-01.yaml b/config/forecasters-co2-mod-01.yaml index b95c6bad..c551a118 100644 --- a/config/forecasters-co2-mod-01.yaml +++ b/config/forecasters-co2-mod-01.yaml @@ -54,3 +54,25 @@ profile: runtime: "23h" gpus: 0 jobs: 50 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/config/forecasters-co2.yaml b/config/forecasters-co2.yaml index 059ab97e..fa7cd811 100644 --- a/config/forecasters-co2.yaml +++ b/config/forecasters-co2.yaml @@ -53,3 +53,25 @@ profile: jobs: 50 batch_rules: plot_forecast_frame: 32 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index 1e9a09b4..2e76d8b6 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -7,8 +7,8 @@ dates: # end: 2025-06-20T00:00 # frequency: 54h # or - - 2024-01-01T12:00 - - 2024-02-01T18:00 + # - 2024-01-01T12:00 + # - 2024-02-01T18:00 - 2025-03-01T00:00 @@ -68,3 +68,25 @@ profile: jobs: 50 batch_rules: plot_forecast_frame: 32 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 6e5b011f..45c5d70d 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -64,3 +64,25 @@ profile: jobs: 50 batch_rules: plot_forecast_frame: 32 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index a5de9b54..4dc1a01d 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -76,3 +76,25 @@ profile: jobs: 50 batch_rules: plot_forecast_frame: 32 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/config/forecasters-ich1_mod_01.yaml b/config/forecasters-ich1_mod_01.yaml index 909e8180..91110a02 100644 --- a/config/forecasters-ich1_mod_01.yaml +++ b/config/forecasters-ich1_mod_01.yaml @@ -55,3 +55,25 @@ profile: runtime: "23h" gpus: 0 jobs: 50 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/config/forecasters-ich1_mod_02_1yr.yaml b/config/forecasters-ich1_mod_02_1yr.yaml index e876f643..52615de1 100644 --- a/config/forecasters-ich1_mod_02_1yr.yaml +++ b/config/forecasters-ich1_mod_02_1yr.yaml @@ -55,3 +55,25 @@ profile: runtime: "23h" gpus: 0 jobs: 50 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/config/interpolators-co2.yaml b/config/interpolators-co2.yaml index 115dd6bc..46cd7a9a 100644 --- a/config/interpolators-co2.yaml +++ b/config/interpolators-co2.yaml @@ -84,3 +84,25 @@ profile: jobs: 50 batch_rules: plot_forecast_frame: 32 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all diff --git a/src/evalml/cli.py b/src/evalml/cli.py index 839e7a21..25dde481 100644 --- a/src/evalml/cli.py +++ b/src/evalml/cli.py @@ -65,6 +65,7 @@ def execute_workflow( unlock: bool, report: Path | None, extra_smk_args: tuple[str, ...] = (), + extra_targets: list[str] = [], ): config = ConfigModel.model_validate(load_yaml(configfile)) @@ -80,7 +81,7 @@ def execute_workflow( if report and not dry_run: command += ["--report-after-run", "--report", str(report)] - command.append(target) + command += [target] + extra_targets command += list(extra_smk_args) if not verbose: command += ["--quiet", "rules"] # reduce verobosity of snakemake output @@ -97,8 +98,14 @@ def cli(): @click.argument( "configfile", type=click.Path(exists=True, dir_okay=False, path_type=Path) ) +@click.option( + "--spatial", + is_flag=True, + default=False, + help="Also run spatial verification (computationally intensive).", +) @workflow_options -def experiment(configfile, cores, verbose, dry_run, unlock, report, extra_smk_args): +def experiment(configfile, spatial, cores, verbose, dry_run, unlock, report, extra_smk_args): execute_workflow( configfile, "experiment_all", @@ -108,6 +115,7 @@ def experiment(configfile, cores, verbose, dry_run, unlock, report, extra_smk_ar unlock, report, extra_smk_args, + extra_targets=["spatial_all"] if spatial else [], ) diff --git a/src/evalml/config.py b/src/evalml/config.py index 802afabb..2ea773a2 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -200,6 +200,34 @@ class BaselineItem(BaseModel): baseline: BaselineConfig +class SpatialVerificationConfig(BaseModel): + """Parameters controlling which spatial verification plots are produced.""" + + params: List[str] = Field( + default=["T_2M", "TD_2M", "U_10M", "V_10M", "SP_10M", "PS", "PMSL", "TOT_PREC"], + description=( + "List of parameters to plot. Supported values: T_2M, TD_2M, U_10M, V_10M, " + "PS, PMSL, TOT_PREC (native), and SP_10M (derived wind speed from U_10M/V_10M)." + ), + ) + leadtimes: List[int] = Field( + default=list(range(6, 121, 6)), + description="List of lead times (hours) to plot.", + ) + metrics: List[str] = Field( + default=["BIAS", "RMSE", "MAE"], + description="List of verification metrics to plot.", + ) + regions: List[str] = Field( + default=["switzerland", "centraleurope"], + description="List of regions to plot.", + ) + seasons: List[str] = Field( + default=["all", "DJF", "MAM", "JJA", "SON"], + description="List of seasons to plot.", + ) + + class Locations(BaseModel): """Locations of data and services used in the workflow.""" @@ -310,6 +338,10 @@ class ConfigModel(BaseModel): stratification: Stratification locations: Locations profile: Profile + spatial_verification: SpatialVerificationConfig = Field( + default_factory=SpatialVerificationConfig, + description="Parameters for spatial verification plots (used with --spatial flag).", + ) model_config = { "extra": "forbid", # fail on misspelled keys diff --git a/workflow/Snakefile b/workflow/Snakefile index edd220ce..d8d6ea11 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -126,26 +126,21 @@ rule experiment_all: rules.verif_metrics_plot.output, experiment=EXPERIMENT_NAME, ), + + +rule spatial_all: + """Target rule for spatial verification (opt-in via evalml experiment --spatial).""" + input: expand( rules.plot_summary_stat_maps.output, run_id=collect_all_candidates(), - leadtime=[6], - metric=["BIAS"], - param=["SP_10M"], - region=["centraleurope", "switzerland"], - season=["all", "DJF", "MAM", "JJA", "SON"], + leadtime=config["spatial_verification"]["leadtimes"], + metric=config["spatial_verification"]["metrics"], + param=config["spatial_verification"]["params"], + region=config["spatial_verification"]["regions"], + season=config["spatial_verification"]["seasons"], experiment=EXPERIMENT_NAME, ), - # expand( - # OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", - # baseline_id=collect_all_baselines(), - # leadtime=[6], - # metric=["BIAS"], - # param=["SP_10M"], - # region=["centraleurope", "switzerland"], - # season=["all", "DJF", "MAM", "JJA", "SON"], - # experiment=EXPERIMENT_NAME - # ) rule showcase_all: From df65a717f23b48237e4a09bef429552fbddd624a Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 2 Apr 2026 11:28:53 +0200 Subject: [PATCH 067/129] Addendum suggested by Claude. --- workflow/tools/config.schema.json | 99 +++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index cc66d7fb..b483ade3 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -458,6 +458,101 @@ "title": "Profile", "type": "object" }, + "SpatialVerificationConfig": { + "description": "Parameters controlling which spatial verification plots are produced.", + "properties": { + "params": { + "default": [ + "T_2M", + "TD_2M", + "U_10M", + "V_10M", + "SP_10M", + "PS", + "PMSL", + "TOT_PREC" + ], + "description": "List of parameters to plot. Supported values: T_2M, TD_2M, U_10M, V_10M, PS, PMSL, TOT_PREC (native), and SP_10M (derived wind speed from U_10M/V_10M).", + "items": { + "type": "string" + }, + "title": "Params", + "type": "array" + }, + "leadtimes": { + "default": [ + 6, + 12, + 18, + 24, + 30, + 36, + 42, + 48, + 54, + 60, + 66, + 72, + 78, + 84, + 90, + 96, + 102, + 108, + 114, + 120 + ], + "description": "List of lead times (hours) to plot.", + "items": { + "type": "integer" + }, + "title": "Leadtimes", + "type": "array" + }, + "metrics": { + "default": [ + "BIAS", + "RMSE", + "MAE" + ], + "description": "List of verification metrics to plot.", + "items": { + "type": "string" + }, + "title": "Metrics", + "type": "array" + }, + "regions": { + "default": [ + "switzerland", + "centraleurope" + ], + "description": "List of regions to plot.", + "items": { + "type": "string" + }, + "title": "Regions", + "type": "array" + }, + "seasons": { + "default": [ + "all", + "DJF", + "MAM", + "JJA", + "SON" + ], + "description": "List of seasons to plot.", + "items": { + "type": "string" + }, + "title": "Seasons", + "type": "array" + } + }, + "title": "SpatialVerificationConfig", + "type": "object" + }, "Stratification": { "description": "Stratification settings for the analysis.", "properties": { @@ -579,6 +674,10 @@ }, "profile": { "$ref": "#/$defs/Profile" + }, + "spatial_verification": { + "$ref": "#/$defs/SpatialVerificationConfig", + "description": "Parameters for spatial verification plots (used with --spatial flag)." } }, "required": [ From be1bbefe818140433c6d09afd9038e0fcd7e8610 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 2 Apr 2026 17:16:58 +0200 Subject: [PATCH 068/129] Add default experiment config Remove later, only for frl development (feel free to use). --- ...l_adj_lfr_05_Stage_E_only_default_exp.yaml | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 config/forecasters-ich1-oper-small_adj_lfr_05_Stage_E_only_default_exp.yaml diff --git a/config/forecasters-ich1-oper-small_adj_lfr_05_Stage_E_only_default_exp.yaml b/config/forecasters-ich1-oper-small_adj_lfr_05_Stage_E_only_default_exp.yaml new file mode 100644 index 00000000..d9051a94 --- /dev/null +++ b/config/forecasters-ich1-oper-small_adj_lfr_05_Stage_E_only_default_exp.yaml @@ -0,0 +1,87 @@ +# yaml-language-server: $schema=../workflow/tools/config.schema.json +description: | + Evaluate skill of ICON-CH1 forecasters from experiment 602 (excluding stage_B, AE, multi_dataset_icon_era5). + +dates: + start: 2025-03-01T00:00 + end: 2025-05-31T00:00 + frequency: 24h + + +runs: + + - forecaster: + inference_resources: + slurm_partition: normal-shared + checkpoint: https://service.meteoswiss.ch/mlstore#/experiments/602/runs/fd63e17043014af59170c7beca516b95 + label: stage_E_realch1 + steps: 0/120/6 + config: resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml + extra_requirements: + - git+https://github.com/ecmwf/anemoi-inference.git@b9aaee5df86614cad9d8d08b76876a4be4e980db + + +baselines: + - baseline: + baseline_id: ICON-CH1-EPS + label: ICON-CH1-ctrl + root: /scratch/mch/cmerker/ICON-CH1-EPS + steps: 0/33/6 + - baseline: + baseline_id: ICON-CH2-EPS + label: ICON-CH2-ctrl + root: /scratch/mch/cmerker/ICON-CH2-EPS + steps: 0/120/6 + +truth: + label: KENDA-CH1 + root: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2025-1h-pl13-v1.0.zarr + +stratification: + regions: + - jura + - mittelland + - voralpen + - alpennordhang + - innerealpentaeler + - alpensuedseite + root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 + +locations: + output_root: /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/xyz_abc/evalml/output/ + +profile: + executor: slurm + global_resources: + gpus: 16 + default_resources: + slurm_partition: "postproc" + cpus_per_task: 1 + mem_mb_per_cpu: 1800 + runtime: "1h" + gpus: 0 + jobs: 50 + batch_rules: + plot_forecast_frame: 32 + +spatial_verification: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - SP_10M + - PS + - PMSL + - TOT_PREC + leadtimes: + - 6 + - 24 + metrics: + - BIAS + - RMSE + regions: + - switzerland + - centraleurope + seasons: + - all From cf4ae6de70d72c98357c0746be7ddca2e2c387c7 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 2 Apr 2026 21:35:55 +0200 Subject: [PATCH 069/129] Spatial Verification Pipeline also for Baselines. --- workflow/Snakefile | 18 +++++-- workflow/rules/plot.smk | 2 +- workflow/rules/verif.smk | 109 ++++++++++++++++++++++++++------------- 3 files changed, 88 insertions(+), 41 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index d8d6ea11..13ea8f05 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -135,10 +135,20 @@ rule spatial_all: rules.plot_summary_stat_maps.output, run_id=collect_all_candidates(), leadtime=config["spatial_verification"]["leadtimes"], - metric=config["spatial_verification"]["metrics"], - param=config["spatial_verification"]["params"], - region=config["spatial_verification"]["regions"], - season=config["spatial_verification"]["seasons"], + metric =config["spatial_verification"]["metrics"], + param =config["spatial_verification"]["params"], + region =config["spatial_verification"]["regions"], + season =config["spatial_verification"]["seasons"], + experiment=EXPERIMENT_NAME, + ), + expand( + rules.plot_summary_stat_maps_baseline.output, + baseline_id=list(BASELINE_CONFIGS), + leadtime=config["spatial_verification"]["leadtimes"], + metric =config["spatial_verification"]["metrics"], + param =config["spatial_verification"]["params"], + region =config["spatial_verification"]["regions"], + season =config["spatial_verification"]["seasons"], experiment=EXPERIMENT_NAME, ), diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index c8550d4d..7a15fddb 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -174,7 +174,7 @@ rule plot_summary_stat_maps: use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: input: script="workflow/scripts/plot_summary_stat_maps.mo.py", - verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", + verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_spatial/{param}_{leadtime}.nc", output: OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", log: diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index 13a2970f..d4d0dba6 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -96,42 +96,6 @@ rule verif_metrics: --output {output} > {log} 2>&1 """ -rule verif_metrics_spatial: - input: - "src/verification/__init__.py", - "src/data_input/__init__.py", - script="workflow/scripts/verif_spatial.py", - inference_okfiles=lambda wc: expand( - rules.execute_inference.output.okfile, - init_time=_restrict_reftimes_to_hours(REFTIMES), - allow_missing=True, - ), - truth=config["truth"]["root"], - output: - OUT_ROOT / "data/runs/{run_id}/verif_spatial/{param}_{leadtime}.nc", - # wildcard_constraints: - # run_id="^" # to avoid ambiguitiy with run_baseline_verif - # TODO: implement logic to use experiment name instead of run_id as wildcard - params: - fcst_label=lambda wc: RUN_CONFIGS[wc.run_id].get("label"), - fcst_steps=lambda wc: RUN_CONFIGS[wc.run_id]["steps"], - truth_label=config["truth"]["label"], - log: - OUT_ROOT / "logs/verif_metrics_spatial/{run_id}-{param}-{leadtime}.log", - resources: - cpus_per_task=24, - mem_mb=50_000, - runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" - shell: - """ - uv run workflow/scripts/verif_spatial.py \ - --run_root output/data/runs/{wildcards.run_id} \ - --truth {input.truth} \ - --step {wildcards.leadtime} \ - --param {wildcards.param} \ - --output {output} > {log} 2>&1 - """ def _restrict_reftimes_to_hours(reftimes, hours=None): @@ -199,3 +163,76 @@ rule verif_metrics_plot: """ uv run {input.script} {input.verif} --output_dir {output} > {log} 2>&1 """ + +rule verif_metrics_spatial: + input: + "src/verification/__init__.py", + "src/data_input/__init__.py", + script="workflow/scripts/verif_spatial.py", + inference_okfiles=lambda wc: expand( + rules.execute_inference.output.okfile, + init_time=_restrict_reftimes_to_hours(REFTIMES), + allow_missing=True, + ), + truth=config["truth"]["root"], + output: + OUT_ROOT / "data/runs/{run_id}/verif_spatial/{param}_{leadtime}.nc", + # wildcard_constraints: + # run_id="^" # to avoid ambiguitiy with run_baseline_verif + # TODO: implement logic to use experiment name instead of run_id as wildcard + params: + fcst_label=lambda wc: RUN_CONFIGS[wc.run_id].get("label"), + fcst_steps=lambda wc: RUN_CONFIGS[wc.run_id]["steps"], + truth_label=config["truth"]["label"], + log: + OUT_ROOT / "logs/verif_metrics_spatial/{run_id}-{param}-{leadtime}.log", + resources: + cpus_per_task=24, + mem_mb=50_000, + runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + shell: + """ + uv run {input.script} \ + --run_root output/data/runs/{wildcards.run_id} \ + --truth {input.truth} \ + --step {wildcards.leadtime} \ + --param {wildcards.param} \ + --output {output} > {log} 2>&1 + """ + +rule verif_metrics_spatial_baseline: + input: + script="workflow/scripts/verif_spatial_baseline.py", + baseline_zarrs=lambda wc: expand( + "{root}/FCST{year}.zarr", + root=BASELINE_CONFIGS[wc.baseline_id].get("root"), + year=sorted({t.strftime("%y") for t in REFTIMES}), + ), + truth=config["truth"]["root"], + output: + OUT_ROOT / "data/baselines/{baseline_id}/verif_spatial/{param}_{leadtime}.nc", + params: + baseline_label=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("label"), + baseline_steps=lambda wc: BASELINE_CONFIGS[wc.baseline_id]["steps"], + baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), + truth_label=config["truth"]["label"], + log: + OUT_ROOT / "logs/verif_metrics_spatial_baseline/{baseline_id}-{param}-{leadtime}.log", + resources: + cpus_per_task=24, + mem_mb=50_000, + runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + shell: + """ + uv run {input.script} \ + --baseline_root {params.baseline_root} \ + --truth {input.truth} \ + --step {wildcards.leadtime} \ + --param {wildcards.param} \ + --label "{params.baseline_label}" \ + --steps "{params.baseline_steps}" \ + --truth_label "{params.truth_label}" \ + --output {output} > {log} 2>&1 + """ From c6a392d39387561837d8981fba94c2c1e3479a08 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 2 Apr 2026 22:00:02 +0200 Subject: [PATCH 070/129] Cosmetic Change. --- workflow/Snakefile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 13ea8f05..49c1a427 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -134,21 +134,21 @@ rule spatial_all: expand( rules.plot_summary_stat_maps.output, run_id=collect_all_candidates(), - leadtime=config["spatial_verification"]["leadtimes"], - metric =config["spatial_verification"]["metrics"], - param =config["spatial_verification"]["params"], - region =config["spatial_verification"]["regions"], - season =config["spatial_verification"]["seasons"], + leadtime = config["spatial_verification"]["leadtimes"], + metric = config["spatial_verification"]["metrics"], + param = config["spatial_verification"]["params"], + region = config["spatial_verification"]["regions"], + season = config["spatial_verification"]["seasons"], experiment=EXPERIMENT_NAME, ), expand( rules.plot_summary_stat_maps_baseline.output, baseline_id=list(BASELINE_CONFIGS), - leadtime=config["spatial_verification"]["leadtimes"], - metric =config["spatial_verification"]["metrics"], - param =config["spatial_verification"]["params"], - region =config["spatial_verification"]["regions"], - season =config["spatial_verification"]["seasons"], + leadtime = config["spatial_verification"]["leadtimes"], + metric = config["spatial_verification"]["metrics"], + param = config["spatial_verification"]["params"], + region = config["spatial_verification"]["regions"], + season = config["spatial_verification"]["seasons"], experiment=EXPERIMENT_NAME, ), From 8a603015fceb38070544a9d359b292f412f727fb Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 30 Apr 2026 11:57:01 +0200 Subject: [PATCH 071/129] Unify run and baseline spatial verification, restrict to configured init times The script now supports both run (GRIB) and baseline (zarr) inputs via mutually exclusive --run_root / --baseline_root flags, replacing the placeholder verif_spatial_baseline.py. The baseline rule drops the unused label/steps/truth_label params and gains --baseline_zarrs for dependency tracking. A new --reftimes flag restricts processing to the experiment's hindcast period. Without it, baseline zarrs (which contain a continuous archive) cause the script to iterate every init time in the file. Both rules pass REFTIMES; the runs side also benefits as a defensive guard against stale on-disk directories. --- workflow/rules/verif.smk | 13 ++- workflow/scripts/verif_spatial.py | 133 ++++++++++++++++++++++-------- 2 files changed, 103 insertions(+), 43 deletions(-) diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index d4d0dba6..adfd58b3 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -184,6 +184,7 @@ rule verif_metrics_spatial: fcst_label=lambda wc: RUN_CONFIGS[wc.run_id].get("label"), fcst_steps=lambda wc: RUN_CONFIGS[wc.run_id]["steps"], truth_label=config["truth"]["label"], + reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: OUT_ROOT / "logs/verif_metrics_spatial/{run_id}-{param}-{leadtime}.log", resources: @@ -195,6 +196,7 @@ rule verif_metrics_spatial: """ uv run {input.script} \ --run_root output/data/runs/{wildcards.run_id} \ + --reftimes {params.reftimes} \ --truth {input.truth} \ --step {wildcards.leadtime} \ --param {wildcards.param} \ @@ -203,7 +205,7 @@ rule verif_metrics_spatial: rule verif_metrics_spatial_baseline: input: - script="workflow/scripts/verif_spatial_baseline.py", + script="workflow/scripts/verif_spatial.py", baseline_zarrs=lambda wc: expand( "{root}/FCST{year}.zarr", root=BASELINE_CONFIGS[wc.baseline_id].get("root"), @@ -213,10 +215,8 @@ rule verif_metrics_spatial_baseline: output: OUT_ROOT / "data/baselines/{baseline_id}/verif_spatial/{param}_{leadtime}.nc", params: - baseline_label=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("label"), - baseline_steps=lambda wc: BASELINE_CONFIGS[wc.baseline_id]["steps"], baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), - truth_label=config["truth"]["label"], + reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: OUT_ROOT / "logs/verif_metrics_spatial_baseline/{baseline_id}-{param}-{leadtime}.log", resources: @@ -228,11 +228,10 @@ rule verif_metrics_spatial_baseline: """ uv run {input.script} \ --baseline_root {params.baseline_root} \ + --baseline_zarrs {input.baseline_zarrs} \ + --reftimes {params.reftimes} \ --truth {input.truth} \ --step {wildcards.leadtime} \ --param {wildcards.param} \ - --label "{params.baseline_label}" \ - --steps "{params.baseline_steps}" \ - --truth_label "{params.truth_label}" \ --output {output} > {log} 2>&1 """ diff --git a/workflow/scripts/verif_spatial.py b/workflow/scripts/verif_spatial.py index 28e8911d..fad0b5e9 100644 --- a/workflow/scripts/verif_spatial.py +++ b/workflow/scripts/verif_spatial.py @@ -24,7 +24,7 @@ import numpy as np import xarray as xr -from data_input import load_fct_data_from_grib +from data_input import load_fct_data_from_grib, load_baseline_from_zarr from verification.spatial import map_forecast_to_truth LOG = logging.getLogger(__name__) @@ -185,6 +185,24 @@ def iter_init_dirs(run_root: Path) -> list[tuple[datetime, Path]]: return result +def iter_baseline_init_times(zarr_paths: list[Path], step: int) -> list[datetime]: + """Return all init times from baseline zarr(s) that have the requested step available.""" + step_td = np.timedelta64(step, "h") + reftimes = [] + for zarr_path in zarr_paths: + if not zarr_path.exists(): + LOG.warning("Baseline zarr not found: %s", zarr_path) + continue + ds = xr.open_zarr(zarr_path, consolidated=True, decode_timedelta=True) + if step_td not in ds["step"].values: + LOG.warning("Step %dh not in %s, skipping", step, zarr_path) + continue + for rt in ds["forecast_reference_time"].values: + ts = (rt - np.datetime64("1970-01-01T00:00:00")) / np.timedelta64(1, "s") + reftimes.append(datetime.utcfromtimestamp(float(ts))) + return sorted(reftimes) + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -209,8 +227,21 @@ def main(args: Namespace) -> None: truth_times = set(truth_da.time.values) # keep as datetime64, tolist() yields ints for ns precision LOG.info("Truth opened lazily: %s", truth_da) - init_dirs = iter_init_dirs(args.run_root) - LOG.info("Found %d init time directories", len(init_dirs)) + if args.baseline_root: + init_items = [(rt, None) for rt in iter_baseline_init_times(args.baseline_zarrs, args.step)] + LOG.info("Found %d baseline init times", len(init_items)) + else: + init_items = iter_init_dirs(args.run_root) + LOG.info("Found %d init time directories", len(init_items)) + + # Restrict to the experiment's configured init times if provided. + # Without this, baseline zarrs (which contain a continuous archive) would + # cause the script to process every init time in the file rather than + # only those in the user's hindcast period. + if args.reftimes: + wanted = {datetime.strptime(s, DATETIME_FMT) for s in args.reftimes} + init_items = [(rt, d) for rt, d in init_items if rt in wanted] + LOG.info("Filtered to %d init times matching --reftimes", len(init_items)) step_td = timedelta(hours=args.step) @@ -226,7 +257,7 @@ def main(args: Namespace) -> None: n_ok = 0 n_skip = 0 - for reftime, grib_dir in init_dirs: + for reftime, grib_dir in init_items: valid_time = np.datetime64(reftime + step_td).astype("datetime64[ns]") if valid_time not in truth_times: @@ -243,31 +274,38 @@ def main(args: Namespace) -> None: first_iter = n_ok == 0 # --- load forecast --- - grib_params = list(_DERIVED[args.param]) if args.param in _DERIVED else [args.param] - - # Cumulative params (e.g. TOT_PREC) are disaggregated via diff inside - # load_fct_data_from_grib, so we need to also load the preceding step - # to get a valid result at the requested step. - if args.param in _CUMULATIVE_PARAMS: - prev_step = _preceding_step(grib_dir, args.step) - if prev_step is None: - LOG.warning("No preceding step found for cumulative param %s at step %d, skipping %s", - args.param, args.step, reftime) - n_skip += 1 - continue - load_steps = [prev_step, args.step] - else: - load_steps = [args.step] + fct_params = list(_DERIVED[args.param]) if args.param in _DERIVED else [args.param] try: - fcst = load_fct_data_from_grib( - root=grib_dir, - reftime=reftime, - steps=load_steps, - params=grib_params, - ) + if args.baseline_root: + zarr_path = args.baseline_root / f"FCST{reftime.strftime('%y')}.zarr" + fcst = load_baseline_from_zarr( + root=zarr_path, + reftime=reftime, + steps=[args.step], + params=fct_params, + ) + else: + # Cumulative params (e.g. TOT_PREC) are disaggregated via diff inside + # load_fct_data_from_grib, so we need to also load the preceding step. + if args.param in _CUMULATIVE_PARAMS: + prev_step = _preceding_step(grib_dir, args.step) + if prev_step is None: + LOG.warning("No preceding step for cumulative param %s at step %d, skipping %s", + args.param, args.step, reftime) + n_skip += 1 + continue + load_steps = [prev_step, args.step] + else: + load_steps = [args.step] + fcst = load_fct_data_from_grib( + root=grib_dir, + reftime=reftime, + steps=load_steps, + params=fct_params, + ) except Exception as exc: - LOG.warning("Could not load GRIB for %s: %s", reftime, exc) + LOG.warning("Could not load forecast for %s: %s", reftime, exc) n_skip += 1 continue @@ -404,7 +442,7 @@ def _seasonal_da(compute_fn) -> xr.DataArray: attrs={ "param": args.param, "step_h": args.step, - "run_root": str(args.run_root), + "source": str(args.baseline_root if args.baseline_root else args.run_root), "n_processed": n_ok, "n_skipped": n_skip, }, @@ -419,15 +457,30 @@ def _seasonal_da(compute_fn) -> xr.DataArray: if __name__ == "__main__": parser = ArgumentParser( description=( - "Compute spatial maps of temporally-aggregated forecast errors " - "by streaming over GRIB files from a model run." + "Compute spatial maps of temporally-aggregated forecast errors. " + "Supports both model runs (GRIB) and baselines (zarr). " + "Exactly one of --run_root or --baseline_root must be provided." ) ) parser.add_argument( "--run_root", type=Path, + default=None, help="Root directory of a model run (e.g. output/data/runs/).", ) + parser.add_argument( + "--baseline_root", + type=Path, + default=None, + help="Root directory of a baseline (e.g. /path/to/ICON-CH1-EPS), containing FCST.zarr files.", + ) + parser.add_argument( + "--baseline_zarrs", + type=Path, + nargs="+", + default=None, + help="Explicit list of baseline zarr paths (used by Snakemake for dependency tracking).", + ) parser.add_argument( "--truth", type=Path, @@ -447,19 +500,27 @@ def _seasonal_da(compute_fn) -> xr.DataArray: help="Variable to verify (e.g. T_2M, TD_2M, U_10M).", ) parser.add_argument( - "--output", - type=Path, + "--reftimes", + nargs="+", default=None, help=( - "Output NetCDF file. " - "Default: /verif_spatial__steph.nc" + "Optional list of init times (YYYYMMDDHHMM) to restrict processing to. " + "Required for baselines whose zarr contains a continuous archive." ), ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Output NetCDF file.", + ) args = parser.parse_args() + if bool(args.run_root) == bool(args.baseline_root): + parser.error("Exactly one of --run_root or --baseline_root must be provided.") + if args.output is None: - args.output = ( - args.run_root / f"verif_spatial_{args.param}_step{args.step:03d}h.nc" - ) + source = args.run_root or args.baseline_root + args.output = source / f"verif_spatial_{args.param}_step{args.step:03d}h.nc" main(args) From 50eaeff3aa6d87883297fef2057d77622071a755 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 30 Apr 2026 11:58:08 +0200 Subject: [PATCH 072/129] Todo for future improvement of GRIB loading speed (profiling pending). --- src/data_input/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index e67816fb..0ca62b53 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -107,6 +107,11 @@ def load_fct_data_from_grib( root: Path, reftime: datetime, steps: list[int], params: list[str] ) -> xr.Dataset: """Load forecast data from GRIB files for a specific valid time.""" + # TODO: this function carries a large per-call setup cost that is + # independent of data volume (likely eccodes/FileDataSource init or GRIB + # index build, not decoding). It dominates runtime in any rule that calls + # it inside a per-init-time loop (e.g. verif_metrics_spatial) and also + # adds noticeable overhead to verif_metrics and the plot rules. files = sorted(root.glob(f"{reftime:%Y%m%d%H%M}*.grib")) fds = data_source.FileDataSource(datafiles=files) ds = grib_decoder.load(fds, {"param": params, "step": steps}) From 75bbf6c0c1367a5cf78cb460b925eefbe4413185 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 30 Apr 2026 12:15:06 +0200 Subject: [PATCH 073/129] Move spatial map plots under plots/maps/ Map outputs previously lived at results/{experiment}/metrics/spatial/... which conflated "spatial verification metrics" (a broader category) with "map plots" (one specific output type) and split rendered outputs across two top-level directories. Consolidating under plots/maps/{runs,baselines}/ groups all rendered figures together and frees the "spatial" name for future use. --- workflow/rules/plot.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 7a15fddb..0c952772 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -147,7 +147,7 @@ rule plot_summary_stat_maps: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/runs/{run_id}/verif_spatial/{param}_{leadtime}.nc", output: - OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/plots/maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits log: @@ -176,7 +176,7 @@ use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_spatial/{param}_{leadtime}.nc", output: - OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/plots/maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", log: OUT_ROOT / "logs/plot_summary_stat_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", params: From 0362d61881c2e7c053f7629b0562308f52217d66 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 4 May 2026 17:52:22 +0200 Subject: [PATCH 074/129] Drop preprocess_field cell and TOT_PREC quick-fix from spatial map plot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The commented preprocess_field cell (carried over from plot_forecast_frame) adds no value here: BIAS/RMSE/MAE are computed on differences, so K↔°C is a no-op on values; SP_10M is already derived upstream in verif_spatial.py; m/s↔knots is unused. The TOT_PREC m→mm quick fix is removed too — that conversion is being handled centrally in a separate PR. --- workflow/scripts/plot_summary_stat_maps.mo.py | 83 +------------------ 1 file changed, 1 insertion(+), 82 deletions(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 50d2d414..f7cde178 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -129,76 +129,6 @@ def get_style(param, metric, units_override=None): } return (get_style,) -# @app.cell -# def _(LOG, np): -# """Preprocess fields with pint-based unit conversion and derived quantities.""" -# try: -# import pint # type: ignore - -# _ureg = pint.UnitRegistry() - -# def _k_to_c(arr): -# # robust conversion with pint, fallback if dtype unsupported -# try: -# return (_ureg.Quantity(arr, _ureg.kelvin).to(_ureg.degC)).magnitude -# except Exception: -# return arr - 273.15 - -# def _ms_to_knots(arr): -# # robust conversion with pint, fallback if dtype unsupported -# try: -# return ( -# _ureg.Quantity(arr, _ureg.meter / _ureg.second).to(_ureg.knot) -# ).magnitude -# except Exception: -# return arr * 1.943844 - -# def _m_to_mm(arr): -# # robust conversion with pint, fallback if dtype unsupported -# try: -# return (_ureg.Quantity(arr, _ureg.meter).to(_ureg.millimeter)).magnitude -# except Exception: -# return arr * 1000 - -# except Exception: -# LOG.warning("pint not available; falling back hardcoded conversions") - -# def _k_to_c(arr): -# return arr - 273.15 - -# def _ms_to_knots(arr): -# return arr * 1.943844 - -# def _m_to_mm(arr): -# return arr * 1000 - -# def preprocess_field(param: str, state: dict): -# """ -# - Temperatures: K -> °C -# - Wind speed: sqrt(u^2 + v^2) -# - Precipitation: m -> mm -# Returns: (field_array, units_override or None) -# """ -# fields = state["fields"] -# # temperature variables -# if param in ("T_2M", "TD_2M", "T", "TD"): -# return _k_to_c(fields[param]), "°C" -# # 10m wind speed (allow legacy 'uv' alias) -# if param == "SP_10M": -# u = fields["U_10M"] -# v = fields["V_10M"] -# return np.sqrt(u**2 + v**2), "m/s" -# # wind speed from standard-level components -# if param == "SP": -# u = fields["U"] -# v = fields["V"] -# return np.sqrt(u**2 + v**2), "m/s" -# if param == "TOT_PREC": -# return _m_to_mm(fields[param]), "mm" -# # default: passthrough -# return fields[param], None - -# return (preprocess_field,) @app.cell def _( @@ -234,18 +164,7 @@ def _( ) subplot = fig.add_map(row=0, column=0) - # # preprocess field (unit conversion, derived quantities) - # field, units_override = preprocess_field(param, state) - - # Quick fix for precipitation (might have to use preprocess_field in the end (see above)) - # for wind speed, preprocess_field only has conversion from m/s to knots - # (not the other way around), so I assume the values are in m/s - if param == "TOT_PREC": - plot_vals = ds.values.ravel()*1000 - else: - plot_vals = ds.values.ravel() - LOG.info("plot_vals: shape=%s, min=%.4g, max=%.4g, n_nan=%d", - plot_vals.shape, float(np.nanmin(plot_vals)), float(np.nanmax(plot_vals)), int(np.isnan(plot_vals).sum())) + plot_vals = ds.values.ravel() style_kwargs = get_style(param, metric) LOG.info("style_kwargs: %s", style_kwargs) From 4822d3b85102291b9ea1a3c6efba090863a6dfa2 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 4 May 2026 18:27:24 +0200 Subject: [PATCH 075/129] Rename and re-design the map-plotting section of all example configs The metric_maps section of each config now lists all the possible options in each category, but leaves all but one commented out. This way, the user sees all the options and can uncomment as needed. This is done for all subsections except for leadtime, where completion by the user is self-evident. --- config/forecasters-co1e.yaml | 26 ++++++++++++++---------- config/forecasters-co2-disentangled.yaml | 26 ++++++++++++++---------- config/forecasters-co2.yaml | 26 ++++++++++++++---------- config/forecasters-ich1-oper-fixed.yaml | 26 ++++++++++++++---------- config/forecasters-ich1-oper.yaml | 26 ++++++++++++++---------- config/forecasters-ich1.yaml | 26 ++++++++++++++---------- config/interpolators-co2.yaml | 26 ++++++++++++++---------- 7 files changed, 105 insertions(+), 77 deletions(-) diff --git a/config/forecasters-co1e.yaml b/config/forecasters-co1e.yaml index 1a1bac39..4ffe3ba1 100644 --- a/config/forecasters-co1e.yaml +++ b/config/forecasters-co1e.yaml @@ -58,24 +58,28 @@ profile: batch_rules: plot_forecast_frame: 32 -spatial_verification: +metric_maps: params: - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC leadtimes: - - 6 - 24 metrics: - BIAS - - RMSE + # - RMSE + # - MAE regions: - switzerland - - centraleurope + # - centraleurope seasons: - all + # - DJF + # - MAM + # - JJA + # - SON diff --git a/config/forecasters-co2-disentangled.yaml b/config/forecasters-co2-disentangled.yaml index e962767a..aa775e73 100644 --- a/config/forecasters-co2-disentangled.yaml +++ b/config/forecasters-co2-disentangled.yaml @@ -77,24 +77,28 @@ profile: gpus: 0 jobs: 50 -spatial_verification: +metric_maps: params: - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC leadtimes: - - 6 - 24 metrics: - BIAS - - RMSE + # - RMSE + # - MAE regions: - switzerland - - centraleurope + # - centraleurope seasons: - all + # - DJF + # - MAM + # - JJA + # - SON diff --git a/config/forecasters-co2.yaml b/config/forecasters-co2.yaml index fa7cd811..68c3f4ba 100644 --- a/config/forecasters-co2.yaml +++ b/config/forecasters-co2.yaml @@ -54,24 +54,28 @@ profile: batch_rules: plot_forecast_frame: 32 -spatial_verification: +metric_maps: params: - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC leadtimes: - - 6 - 24 metrics: - BIAS - - RMSE + # - RMSE + # - MAE regions: - switzerland - - centraleurope + # - centraleurope seasons: - all + # - DJF + # - MAM + # - JJA + # - SON diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index 2e76d8b6..54ce33e9 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -69,24 +69,28 @@ profile: batch_rules: plot_forecast_frame: 32 -spatial_verification: +metric_maps: params: - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC leadtimes: - - 6 - 24 metrics: - BIAS - - RMSE + # - RMSE + # - MAE regions: - switzerland - - centraleurope + # - centraleurope seasons: - all + # - DJF + # - MAM + # - JJA + # - SON diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 45c5d70d..0e98b66c 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -65,24 +65,28 @@ profile: batch_rules: plot_forecast_frame: 32 -spatial_verification: +metric_maps: params: - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC leadtimes: - - 6 - 24 metrics: - BIAS - - RMSE + # - RMSE + # - MAE regions: - switzerland - - centraleurope + # - centraleurope seasons: - all + # - DJF + # - MAM + # - JJA + # - SON diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index 4dc1a01d..1692513c 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -77,24 +77,28 @@ profile: batch_rules: plot_forecast_frame: 32 -spatial_verification: +metric_maps: params: - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC leadtimes: - - 6 - 24 metrics: - BIAS - - RMSE + # - RMSE + # - MAE regions: - switzerland - - centraleurope + # - centraleurope seasons: - all + # - DJF + # - MAM + # - JJA + # - SON diff --git a/config/interpolators-co2.yaml b/config/interpolators-co2.yaml index 46cd7a9a..77573ca6 100644 --- a/config/interpolators-co2.yaml +++ b/config/interpolators-co2.yaml @@ -85,24 +85,28 @@ profile: batch_rules: plot_forecast_frame: 32 -spatial_verification: +metric_maps: params: - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC leadtimes: - - 6 - 24 metrics: - BIAS - - RMSE + # - RMSE + # - MAE regions: - switzerland - - centraleurope + # - centraleurope seasons: - all + # - DJF + # - MAM + # - JJA + # - SON From d1ca1225318e79a1a747a9cd695333c091a818b3 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 4 May 2026 19:07:30 +0200 Subject: [PATCH 076/129] =?UTF-8?q?Rename=20"spatial=20verification"=20?= =?UTF-8?q?=E2=86=92=20"metric=20maps"=20throughout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The config section "spatial_verification" overstates scope; the feature is specifically about temporally-aggregated metric map plots, not the broader category of spatial verification techniques. Rename the user- facing surface and the internal naming chain consistently: - Config: MetricMapsConfig / config["metric_maps"] - CLI: --maps (was --spatial) - Target rule: metric_maps_all (was spatial_all) - Rules: verif_metrics_maps[_baseline] (was verif_metrics_spatial[_baseline]) - Script: verif_metric_maps.py (was verif_spatial.py) - Output dir: data/.../metric_maps/ (was verif_spatial/) - Log dir: logs/verif_metrics_maps[_baseline]/ - Colormap-defaults keys: *.map (was *.spatial) --- src/data_input/__init__.py | 2 +- src/evalml/cli.py | 8 +- src/evalml/config.py | 10 +-- src/plotting/colormap_defaults.py | 48 +++++----- workflow/Snakefile | 24 ++--- workflow/rules/plot.smk | 4 +- workflow/rules/verif.smk | 16 ++-- workflow/scripts/plot_summary_stat_maps.mo.py | 2 +- ...{verif_spatial.py => verif_metric_maps.py} | 4 +- workflow/tools/config.schema.json | 88 +++++++++---------- 10 files changed, 103 insertions(+), 103 deletions(-) rename workflow/scripts/{verif_spatial.py => verif_metric_maps.py} (99%) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index 0ca62b53..5677d969 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -110,7 +110,7 @@ def load_fct_data_from_grib( # TODO: this function carries a large per-call setup cost that is # independent of data volume (likely eccodes/FileDataSource init or GRIB # index build, not decoding). It dominates runtime in any rule that calls - # it inside a per-init-time loop (e.g. verif_metrics_spatial) and also + # it inside a per-init-time loop (e.g. verif_metrics_maps) and also # adds noticeable overhead to verif_metrics and the plot rules. files = sorted(root.glob(f"{reftime:%Y%m%d%H%M}*.grib")) fds = data_source.FileDataSource(datafiles=files) diff --git a/src/evalml/cli.py b/src/evalml/cli.py index 25dde481..af2cd27c 100644 --- a/src/evalml/cli.py +++ b/src/evalml/cli.py @@ -99,13 +99,13 @@ def cli(): "configfile", type=click.Path(exists=True, dir_okay=False, path_type=Path) ) @click.option( - "--spatial", + "--maps", is_flag=True, default=False, - help="Also run spatial verification (computationally intensive).", + help="Also produce metric maps (computationally intensive).", ) @workflow_options -def experiment(configfile, spatial, cores, verbose, dry_run, unlock, report, extra_smk_args): +def experiment(configfile, maps, cores, verbose, dry_run, unlock, report, extra_smk_args): execute_workflow( configfile, "experiment_all", @@ -115,7 +115,7 @@ def experiment(configfile, spatial, cores, verbose, dry_run, unlock, report, ext unlock, report, extra_smk_args, - extra_targets=["spatial_all"] if spatial else [], + extra_targets=["metric_maps_all"] if maps else [], ) diff --git a/src/evalml/config.py b/src/evalml/config.py index 2ea773a2..68d20b75 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -200,8 +200,8 @@ class BaselineItem(BaseModel): baseline: BaselineConfig -class SpatialVerificationConfig(BaseModel): - """Parameters controlling which spatial verification plots are produced.""" +class MetricMapsConfig(BaseModel): + """Parameters controlling which metric map plots are produced.""" params: List[str] = Field( default=["T_2M", "TD_2M", "U_10M", "V_10M", "SP_10M", "PS", "PMSL", "TOT_PREC"], @@ -338,9 +338,9 @@ class ConfigModel(BaseModel): stratification: Stratification locations: Locations profile: Profile - spatial_verification: SpatialVerificationConfig = Field( - default_factory=SpatialVerificationConfig, - description="Parameters for spatial verification plots (used with --spatial flag).", + metric_maps: MetricMapsConfig = Field( + default_factory=MetricMapsConfig, + description="Parameters for metric map plots (used with --maps flag).", ) model_config = { diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 7ad4878a..5cb2ea03 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -52,39 +52,39 @@ def _fallback(): # always start at 0 so that the saturation of the colour corresponds to the error magnitude. # RMSE: - "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "SP_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, - "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, - "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, + "U_10M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "V_10M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "SP_10M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "TD_2M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "T_2M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "PMSL.RMSE.map": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "PS.RMSE.map": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "TOT_PREC.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, # would ideally want a 6th colour on the high end of the colour scale, but somehow # matplotlib does not do that -> ? # MAE: - "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "SP_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, - "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, - "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, + "U_10M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "V_10M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "SP_10M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "TD_2M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "T_2M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "PMSL.MAE.map": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "PS.MAE.map": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "TOT_PREC.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, # the levels for precip are a bit on the bright side, but still worth keeping consistent with RMSE. # Bias: # diverging colour scheme for the Bias to reflect the nature of the data (can be positive or negative, symmetric). # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. - "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, - "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, - "SP_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, - "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, - "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, - "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, - "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, - "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 9), "levels": [-1, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 1]} | {"units": "mm"} + "U_10M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "V_10M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "SP_10M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "TD_2M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, + "T_2M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, + "PMSL.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, + "PS.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, + "TOT_PREC.BIAS.map": {"cmap": plt.get_cmap("BrBG", 9), "levels": [-1, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 1]} | {"units": "mm"} } CMAP_DEFAULTS = defaultdict(_fallback, _CMAP_DEFAULTS) diff --git a/workflow/Snakefile b/workflow/Snakefile index 49c1a427..a5ead0c1 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -128,27 +128,27 @@ rule experiment_all: ), -rule spatial_all: - """Target rule for spatial verification (opt-in via evalml experiment --spatial).""" +rule metric_maps_all: + """Target rule for metric maps (opt-in via evalml experiment --maps).""" input: expand( rules.plot_summary_stat_maps.output, run_id=collect_all_candidates(), - leadtime = config["spatial_verification"]["leadtimes"], - metric = config["spatial_verification"]["metrics"], - param = config["spatial_verification"]["params"], - region = config["spatial_verification"]["regions"], - season = config["spatial_verification"]["seasons"], + leadtime = config["metric_maps"]["leadtimes"], + metric = config["metric_maps"]["metrics"], + param = config["metric_maps"]["params"], + region = config["metric_maps"]["regions"], + season = config["metric_maps"]["seasons"], experiment=EXPERIMENT_NAME, ), expand( rules.plot_summary_stat_maps_baseline.output, baseline_id=list(BASELINE_CONFIGS), - leadtime = config["spatial_verification"]["leadtimes"], - metric = config["spatial_verification"]["metrics"], - param = config["spatial_verification"]["params"], - region = config["spatial_verification"]["regions"], - season = config["spatial_verification"]["seasons"], + leadtime = config["metric_maps"]["leadtimes"], + metric = config["metric_maps"]["metrics"], + param = config["metric_maps"]["params"], + region = config["metric_maps"]["regions"], + season = config["metric_maps"]["seasons"], experiment=EXPERIMENT_NAME, ), diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 0c952772..2ecd9038 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -145,7 +145,7 @@ rule plot_summary_stat_maps: localrule: True input: script="workflow/scripts/plot_summary_stat_maps.mo.py", - verif_file=OUT_ROOT / "data/runs/{run_id}/verif_spatial/{param}_{leadtime}.nc", + verif_file=OUT_ROOT / "data/runs/{run_id}/metric_maps/{param}_{leadtime}.nc", output: OUT_ROOT / "results/{experiment}/plots/maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", wildcard_constraints: @@ -174,7 +174,7 @@ rule plot_summary_stat_maps: use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: input: script="workflow/scripts/plot_summary_stat_maps.mo.py", - verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_spatial/{param}_{leadtime}.nc", + verif_file=OUT_ROOT / "data/baselines/{baseline_id}/metric_maps/{param}_{leadtime}.nc", output: OUT_ROOT / "results/{experiment}/plots/maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", log: diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index adfd58b3..41f6254e 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -164,11 +164,11 @@ rule verif_metrics_plot: uv run {input.script} {input.verif} --output_dir {output} > {log} 2>&1 """ -rule verif_metrics_spatial: +rule verif_metrics_maps: input: "src/verification/__init__.py", "src/data_input/__init__.py", - script="workflow/scripts/verif_spatial.py", + script="workflow/scripts/verif_metric_maps.py", inference_okfiles=lambda wc: expand( rules.execute_inference.output.okfile, init_time=_restrict_reftimes_to_hours(REFTIMES), @@ -176,7 +176,7 @@ rule verif_metrics_spatial: ), truth=config["truth"]["root"], output: - OUT_ROOT / "data/runs/{run_id}/verif_spatial/{param}_{leadtime}.nc", + OUT_ROOT / "data/runs/{run_id}/metric_maps/{param}_{leadtime}.nc", # wildcard_constraints: # run_id="^" # to avoid ambiguitiy with run_baseline_verif # TODO: implement logic to use experiment name instead of run_id as wildcard @@ -186,7 +186,7 @@ rule verif_metrics_spatial: truth_label=config["truth"]["label"], reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: - OUT_ROOT / "logs/verif_metrics_spatial/{run_id}-{param}-{leadtime}.log", + OUT_ROOT / "logs/verif_metrics_maps/{run_id}-{param}-{leadtime}.log", resources: cpus_per_task=24, mem_mb=50_000, @@ -203,9 +203,9 @@ rule verif_metrics_spatial: --output {output} > {log} 2>&1 """ -rule verif_metrics_spatial_baseline: +rule verif_metrics_maps_baseline: input: - script="workflow/scripts/verif_spatial.py", + script="workflow/scripts/verif_metric_maps.py", baseline_zarrs=lambda wc: expand( "{root}/FCST{year}.zarr", root=BASELINE_CONFIGS[wc.baseline_id].get("root"), @@ -213,12 +213,12 @@ rule verif_metrics_spatial_baseline: ), truth=config["truth"]["root"], output: - OUT_ROOT / "data/baselines/{baseline_id}/verif_spatial/{param}_{leadtime}.nc", + OUT_ROOT / "data/baselines/{baseline_id}/metric_maps/{param}_{leadtime}.nc", params: baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: - OUT_ROOT / "logs/verif_metrics_spatial_baseline/{baseline_id}-{param}-{leadtime}.log", + OUT_ROOT / "logs/verif_metrics_maps_baseline/{baseline_id}-{param}-{leadtime}.log", resources: cpus_per_task=24, mem_mb=50_000, diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index f7cde178..9a560330 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -110,7 +110,7 @@ def get_style(param, metric, units_override=None): Needed because cmap/norm does not work in Style(colors=cmap), still needs to be passed as arguments to tripcolor()/tricontourf(). """ - metric_key = f"{param}.{metric}.spatial" + metric_key = f"{param}.{metric}.map" cfg = CMAP_DEFAULTS[metric_key] if metric_key in CMAP_DEFAULTS else CMAP_DEFAULTS.get(param, {}) units = units_override if units_override is not None else cfg.get("units", "") return { diff --git a/workflow/scripts/verif_spatial.py b/workflow/scripts/verif_metric_maps.py similarity index 99% rename from workflow/scripts/verif_spatial.py rename to workflow/scripts/verif_metric_maps.py index fad0b5e9..8c49be3f 100644 --- a/workflow/scripts/verif_spatial.py +++ b/workflow/scripts/verif_metric_maps.py @@ -9,7 +9,7 @@ Usage ----- - uv run workflow/scripts/verif_spatial.py \\ + uv run workflow/scripts/verif_metric_maps.py \\ output/data/runs/ \\ --truth /path/to/truth.zarr \\ --step 24 \\ @@ -521,6 +521,6 @@ def _seasonal_da(compute_fn) -> xr.DataArray: if args.output is None: source = args.run_root or args.baseline_root - args.output = source / f"verif_spatial_{args.param}_step{args.step:03d}h.nc" + args.output = source / f"verif_metric_maps_{args.param}_step{args.step:03d}h.nc" main(args) diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index b483ade3..27222520 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -420,46 +420,8 @@ "title": "Locations", "type": "object" }, - "Profile": { - "description": "Workflow execution profile.", - "properties": { - "executor": { - "description": "Job executor, e.g. 'slurm'.", - "title": "Executor", - "type": "string" - }, - "global_resources": { - "$ref": "#/$defs/GlobalResources" - }, - "default_resources": { - "$ref": "#/$defs/DefaultResources" - }, - "jobs": { - "description": "Maximum number of parallel jobs.", - "minimum": 1, - "title": "Jobs", - "type": "integer" - }, - "batch_rules": { - "additionalProperties": { - "type": "integer" - }, - "description": "Define batches of the same rule that shall be executed within one job submission.", - "title": "Batch Rules", - "type": "object" - } - }, - "required": [ - "executor", - "global_resources", - "default_resources", - "jobs" - ], - "title": "Profile", - "type": "object" - }, - "SpatialVerificationConfig": { - "description": "Parameters controlling which spatial verification plots are produced.", + "MetricMapsConfig": { + "description": "Parameters controlling which metric map plots are produced.", "properties": { "params": { "default": [ @@ -550,7 +512,45 @@ "type": "array" } }, - "title": "SpatialVerificationConfig", + "title": "MetricMapsConfig", + "type": "object" + }, + "Profile": { + "description": "Workflow execution profile.", + "properties": { + "executor": { + "description": "Job executor, e.g. 'slurm'.", + "title": "Executor", + "type": "string" + }, + "global_resources": { + "$ref": "#/$defs/GlobalResources" + }, + "default_resources": { + "$ref": "#/$defs/DefaultResources" + }, + "jobs": { + "description": "Maximum number of parallel jobs.", + "minimum": 1, + "title": "Jobs", + "type": "integer" + }, + "batch_rules": { + "additionalProperties": { + "type": "integer" + }, + "description": "Define batches of the same rule that shall be executed within one job submission.", + "title": "Batch Rules", + "type": "object" + } + }, + "required": [ + "executor", + "global_resources", + "default_resources", + "jobs" + ], + "title": "Profile", "type": "object" }, "Stratification": { @@ -675,9 +675,9 @@ "profile": { "$ref": "#/$defs/Profile" }, - "spatial_verification": { - "$ref": "#/$defs/SpatialVerificationConfig", - "description": "Parameters for spatial verification plots (used with --spatial flag)." + "metric_maps": { + "$ref": "#/$defs/MetricMapsConfig", + "description": "Parameters for metric map plots (used with --maps flag)." } }, "required": [ From ef22a3f80b924e00da27a86909aae9a1c2dd30d1 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 4 May 2026 19:20:41 +0200 Subject: [PATCH 077/129] Remove obsolete spatial-data filtering from dashboard script Three changes accumulated on this branch added increasingly defensive filtering against spatial metric variables in verif_aggregated.nc: the nonspatial_vars filter (a456b1d / 99dac52) and the dataset_metrics_to_dataframe helper plus drop_variables list (7b50809, "Fix memory leak"). All of it was needed when spatial metric maps were written into verif_aggregated.nc and had to be filtered out. In the final architecture, metric maps live in separate data/.../metric_maps/{param}_{leadtime}.nc files and never enter verif_aggregated.nc. The leak cannot occur and the filtering is dead code. Revert to the original inline conversion. --- .../scripts/report_experiment_dashboard.py | 68 +------------------ 1 file changed, 2 insertions(+), 66 deletions(-) diff --git a/workflow/scripts/report_experiment_dashboard.py b/workflow/scripts/report_experiment_dashboard.py index 180035ec..d273e130 100644 --- a/workflow/scripts/report_experiment_dashboard.py +++ b/workflow/scripts/report_experiment_dashboard.py @@ -28,81 +28,17 @@ def program_summary_log(args): LOG.info("=" * 80) -def dataset_metrics_to_dataframe( - ds: xr.Dataset, - forbidden_dims=("values",), - metric_dims=("source", "season", "init_hour", "region", "lead_time", "eps"), -): - """ - Convert a verification xarray.Dataset to a pandas.DataFrame compatible with - the old `to_array("stack").to_dataframe()` workflow. - - Returns columns: - - metric_dims... - - stack (e.g. "T_2M.MAE") - - value - """ - - # keep only non-spatial metric variables - metric_vars = [] - for v in ds.data_vars: - da = ds[v] - - # skip spatial metrics - if "spatial" in v: - continue - - # skip forbidden dimensions - if any(d in da.dims for d in forbidden_dims): - continue - - # only keep vars whose dims are a subset of expected metric dims - if not set(da.dims).issubset(metric_dims): - continue - - metric_vars.append(v) - - if not metric_vars: - raise ValueError("No compatible metric variables found in dataset") - - # stack variables exactly like the original code - df = ( - ds[metric_vars] - .to_array("stack") - .to_dataframe(name="value") - .reset_index() - ) - - return df - def main(args): program_summary_log(args) # Load, de-duplicate lead_time, and keep best provider per source (same logic as verif_plot_metrics) - drop_variables = ["TOT_PREC.MAE.spatial", "TOT_PREC.RMSE.spatial", "TOT_PREC.BIAS.spatial", - "PMSL.MAE.spatial", "PMSL.RMSE.spatial", "PMSL.BIAS.spatial", - "PS.MAE.spatial", "PS.RMSE.spatial", "PS.BIAS.spatial", - "V_10M.MAE.spatial", "V_10M.RMSE.spatial", "V_10M.BIAS.spatial", - "U_10M.MAE.spatial", "U_10M.RMSE.spatial", "U_10M.BIAS.spatial", - "TD_2M.MAE.spatial", "TD_2M.RMSE.spatial", "TD_2M.BIAS.spatial", - "T_2M.MAE.spatial", "T_2M.RMSE.spatial", "T_2M.BIAS.spatial"] - - dfs = [xr.open_dataset(f, drop_variables = drop_variables) for f in args.verif_files] + dfs = [xr.open_dataset(f) for f in args.verif_files] dfs = [_ensure_unique_lead_time(d) for d in dfs] dfs = _select_best_sources(dfs) ds = xr.concat(dfs, dim="source", join="outer") LOG.info("Loaded verification netcdf: \n%s", ds) - # extract only non-spatial variables to pd.DataFrame - # nonspatial_vars = [d for d in ds.data_vars if "spatial" not in d] - # df = ds[nonspatial_vars].to_array("stack").to_dataframe(name="value").reset_index() - - df = dataset_metrics_to_dataframe( - ds, - forbidden_dims=("values",), # critical! - metric_dims=("source", "season", "init_hour", "region", "lead_time", "eps"), - ) - + df = ds.to_array("stack").to_dataframe(name="value").reset_index() df[["param", "metric"]] = df["stack"].str.split(".", n=1, expand=True) df.drop(columns=["stack"], inplace=True) df["lead_time"] = df["lead_time"].dt.total_seconds() / 3600 From 2636813858729f1c65af2e29c40d8697d7ea3b49 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 4 May 2026 19:48:07 +0200 Subject: [PATCH 078/129] Move metric maps to results/{experiment}/metric_maps/ Snakemake refuses outputs that are children of another rule's declared directory output. verif_metrics_plot declares the whole plots/ directory, so plot_summary_stat_maps[_baseline] writing under plots/maps/ caused a ChildIOException. Move the maps to a sibling directory; each top-level dir under results/{experiment}/ now has a single owning rule. --- workflow/rules/plot.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 2ecd9038..dc20bacd 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -147,7 +147,7 @@ rule plot_summary_stat_maps: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/runs/{run_id}/metric_maps/{param}_{leadtime}.nc", output: - OUT_ROOT / "results/{experiment}/plots/maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/metric_maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits log: @@ -176,7 +176,7 @@ use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/baselines/{baseline_id}/metric_maps/{param}_{leadtime}.nc", output: - OUT_ROOT / "results/{experiment}/plots/maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + OUT_ROOT / "results/{experiment}/metric_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", log: OUT_ROOT / "logs/plot_summary_stat_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", params: From b18c3c71834ab216438cc411947e2719b1fba131 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 11:08:56 +0200 Subject: [PATCH 079/129] Remove stray files. --- examine_error_distr_fine_tune.ipynb | 597 -- snakemake_log.txt | 13914 -------------------------- 2 files changed, 14511 deletions(-) delete mode 100644 examine_error_distr_fine_tune.ipynb delete mode 100644 snakemake_log.txt diff --git a/examine_error_distr_fine_tune.ipynb b/examine_error_distr_fine_tune.ipynb deleted file mode 100644 index 5685bb49..00000000 --- a/examine_error_distr_fine_tune.ipynb +++ /dev/null @@ -1,597 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "b7660869", - "metadata": {}, - "source": [ - "Load Verification files and examine the distribution of the spatial components\n", - "to aid fine-tuning of the colour levels for the maps." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "f38d8953", - "metadata": {}, - "outputs": [], - "source": [ - "import xarray as xr\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "eefaf693", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.DataArray 'PS.RMSE.spatial' (eps: 1, values: 1147980)> Size: 5MB\n",
-       "[1147980 values with dtype=float32]\n",
-       "Coordinates:\n",
-       "  * eps        (eps) int64 8B 0\n",
-       "    lead_time  timedelta64[ns] 8B 5 days\n",
-       "    init_hour  int64 8B 0\n",
-       "    season     <U3 12B 'all'\n",
-       "    lon        (values) float64 9MB ...\n",
-       "    lat        (values) float64 9MB ...\n",
-       "    latitude   (values) float64 9MB ...\n",
-       "    longitude  (values) float64 9MB ...\n",
-       "Dimensions without coordinates: values
" - ], - "text/plain": [ - " Size: 5MB\n", - "[1147980 values with dtype=float32]\n", - "Coordinates:\n", - " * eps (eps) int64 8B 0\n", - " lead_time timedelta64[ns] 8B 5 days\n", - " init_hour int64 8B 0\n", - " season " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "# 1. Define and calculate the quantiles\n", - "q_list = [0.2, 0.4, 0.6, 0.8]\n", - "rmse_quantiles = dat_sel.quantile(q_list)\n", - "\n", - "plt.figure(figsize=(12, 7))\n", - "\n", - "# 2. Plot the histogram\n", - "# We set alpha=0.6 so the vertical lines are easier to see\n", - "dat_sel.plot.hist(bins=100, color='skyblue', edgecolor='white', alpha=0.6)\n", - "\n", - "# 3. Add vertical lines for each quantile\n", - "# We'll use a colormap to give each line a different color\n", - "colors = plt.cm.viridis(np.linspace(0, 1, len(q_list)))\n", - "\n", - "for q, val, color in zip(q_list, rmse_quantiles.values, colors):\n", - " # Format the label for the legend: e.g., \"50% (Median): 4.32\"\n", - " prob_label = f\"{int(q*100)}%\"\n", - " if q == 0.5: prob_label += \" (Median)\"\n", - " \n", - " label_text = f\"{prob_label}: {val:.3f}\"\n", - " \n", - " plt.axvline(val, color=color, linestyle='--', linewidth=2, label=label_text)\n", - "\n", - "# 4. Final plot styling\n", - "plt.title(f\"Distribution of {dat_sel.name}\\nwith Statistical Quantiles\", fontsize=14)\n", - "plt.xlabel(\"RMSE Value\", fontsize=12)\n", - "plt.ylabel(\"Frequency (Grid Points)\", fontsize=12)\n", - "\n", - "# Place the legend\n", - "plt.legend(title=\"Quantiles & Values\", loc='upper right', frameon=True, shadow=True)\n", - "\n", - "plt.grid(axis='y', alpha=0.3)\n", - "plt.tight_layout()\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "evalml (3.12.12)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/snakemake_log.txt b/snakemake_log.txt deleted file mode 100644 index 954cbf14..00000000 --- a/snakemake_log.txt +++ /dev/null @@ -1,13914 +0,0 @@ -Launching: snakemake --executor slurm --resources gpus=16 --default-resources slurm_partition=postproc cpus_per_task=1 mem_mb_per_cpu=1800 runtime=23h --jobs 50 --configfile config/forecasters-ich1_mod_02_1yr.yaml --cores 4 --dry-run experiment_all -host: balfrin-ln002 -Building DAG of jobs... -Job stats: -job count ----------------------------------- ------- -execute_inference 359 -experiment_all 1 -make_squashfs_image 1 -plot_summary_stat_maps 10 -plot_summary_stat_maps_baseline 10 -prepare_inference_forecaster 358 -report_experiment_dashboard 1 -verif_metrics 359 -verif_metrics_aggregation 1 -verif_metrics_aggregation_baseline 1 -verif_metrics_baseline 359 -verif_metrics_plot 1 -write_summary 1 -total 1462 - - -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402140000/config.yaml, output/data/runs/b30a-0209f019/202402140000/resources, output/data/runs/b30a-0209f019/202402140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402140000.log - jobid: 497 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401170000/config.yaml, output/data/runs/b30a-0209f019/202401170000/resources, output/data/runs/b30a-0209f019/202401170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401170000.log - jobid: 413 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407310000/config.yaml, output/data/runs/b30a-0209f019/202407310000/resources, output/data/runs/b30a-0209f019/202407310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407310000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407310000.log - jobid: 1001 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408280000/config.yaml, output/data/runs/b30a-0209f019/202408280000/resources, output/data/runs/b30a-0209f019/202408280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408280000.log - jobid: 1085 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405180000.log - jobid: 140 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404100000/config.yaml, output/data/runs/b30a-0209f019/202404100000/resources, output/data/runs/b30a-0209f019/202404100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404100000.log - jobid: 665 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407030000/config.yaml, output/data/runs/b30a-0209f019/202407030000/resources, output/data/runs/b30a-0209f019/202407030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407030000.log - jobid: 917 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409070000.log - jobid: 252 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409250000/config.yaml, output/data/runs/b30a-0209f019/202409250000/resources, output/data/runs/b30a-0209f019/202409250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409250000.log - jobid: 1169 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410230000/config.yaml, output/data/runs/b30a-0209f019/202410230000/resources, output/data/runs/b30a-0209f019/202410230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410230000.log - jobid: 1253 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409080000.log - jobid: 253 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405190000.log - jobid: 141 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401180000/config.yaml, output/data/runs/b30a-0209f019/202401180000/resources, output/data/runs/b30a-0209f019/202401180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401180000.log - jobid: 416 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408010000/config.yaml, output/data/runs/b30a-0209f019/202408010000/resources, output/data/runs/b30a-0209f019/202408010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408010000.log - jobid: 1004 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408290000/config.yaml, output/data/runs/b30a-0209f019/202408290000/resources, output/data/runs/b30a-0209f019/202408290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408290000.log - jobid: 1088 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405200000.log - jobid: 142 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402150000/config.yaml, output/data/runs/b30a-0209f019/202402150000/resources, output/data/runs/b30a-0209f019/202402150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402150000.log - jobid: 500 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404110000/config.yaml, output/data/runs/b30a-0209f019/202404110000/resources, output/data/runs/b30a-0209f019/202404110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404110000.log - jobid: 668 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407040000/config.yaml, output/data/runs/b30a-0209f019/202407040000/resources, output/data/runs/b30a-0209f019/202407040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407040000.log - jobid: 920 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409090000.log - jobid: 254 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409260000/config.yaml, output/data/runs/b30a-0209f019/202409260000/resources, output/data/runs/b30a-0209f019/202409260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409260000.log - jobid: 1172 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410240000/config.yaml, output/data/runs/b30a-0209f019/202410240000/resources, output/data/runs/b30a-0209f019/202410240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410240000.log - jobid: 1256 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405210000.log - jobid: 143 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409100000.log - jobid: 255 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401190000/config.yaml, output/data/runs/b30a-0209f019/202401190000/resources, output/data/runs/b30a-0209f019/202401190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401190000.log - jobid: 419 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408020000/config.yaml, output/data/runs/b30a-0209f019/202408020000/resources, output/data/runs/b30a-0209f019/202408020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408020000.log - jobid: 1007 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408300000/config.yaml, output/data/runs/b30a-0209f019/202408300000/resources, output/data/runs/b30a-0209f019/202408300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408300000.log - jobid: 1091 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405220000.log - jobid: 144 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402160000/config.yaml, output/data/runs/b30a-0209f019/202402160000/resources, output/data/runs/b30a-0209f019/202402160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402160000.log - jobid: 503 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404120000/config.yaml, output/data/runs/b30a-0209f019/202404120000/resources, output/data/runs/b30a-0209f019/202404120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404120000.log - jobid: 671 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407050000/config.yaml, output/data/runs/b30a-0209f019/202407050000/resources, output/data/runs/b30a-0209f019/202407050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407050000.log - jobid: 923 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409110000.log - jobid: 256 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409270000/config.yaml, output/data/runs/b30a-0209f019/202409270000/resources, output/data/runs/b30a-0209f019/202409270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409270000.log - jobid: 1175 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410250000/config.yaml, output/data/runs/b30a-0209f019/202410250000/resources, output/data/runs/b30a-0209f019/202410250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410250000.log - jobid: 1259 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405230000.log - jobid: 145 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409120000.log - jobid: 257 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401200000/config.yaml, output/data/runs/b30a-0209f019/202401200000/resources, output/data/runs/b30a-0209f019/202401200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401200000.log - jobid: 422 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408030000/config.yaml, output/data/runs/b30a-0209f019/202408030000/resources, output/data/runs/b30a-0209f019/202408030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408030000.log - jobid: 1010 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408310000/config.yaml, output/data/runs/b30a-0209f019/202408310000/resources, output/data/runs/b30a-0209f019/202408310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408310000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408310000.log - jobid: 1094 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405240000.log - jobid: 146 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402170000/config.yaml, output/data/runs/b30a-0209f019/202402170000/resources, output/data/runs/b30a-0209f019/202402170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402170000.log - jobid: 506 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404130000/config.yaml, output/data/runs/b30a-0209f019/202404130000/resources, output/data/runs/b30a-0209f019/202404130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404130000.log - jobid: 674 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407060000/config.yaml, output/data/runs/b30a-0209f019/202407060000/resources, output/data/runs/b30a-0209f019/202407060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407060000.log - jobid: 926 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409130000.log - jobid: 258 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409280000/config.yaml, output/data/runs/b30a-0209f019/202409280000/resources, output/data/runs/b30a-0209f019/202409280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409280000.log - jobid: 1178 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410260000/config.yaml, output/data/runs/b30a-0209f019/202410260000/resources, output/data/runs/b30a-0209f019/202410260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410260000.log - jobid: 1262 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405250000.log - jobid: 147 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409140000.log - jobid: 259 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401210000/config.yaml, output/data/runs/b30a-0209f019/202401210000/resources, output/data/runs/b30a-0209f019/202401210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401210000.log - jobid: 425 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408040000/config.yaml, output/data/runs/b30a-0209f019/202408040000/resources, output/data/runs/b30a-0209f019/202408040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408040000.log - jobid: 1013 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409010000/config.yaml, output/data/runs/b30a-0209f019/202409010000/resources, output/data/runs/b30a-0209f019/202409010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409010000.log - jobid: 1097 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule write_summary: - input: /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/config/forecasters-ich1_mod_02_1yr.yaml - output: output/data/runs/b30a-0209f019/summary.md - jobid: 1444 - reason: Set of input files has changed since last execution - wildcards: run_id=b30a-0209f019 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405260000.log - jobid: 148 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402180000/config.yaml, output/data/runs/b30a-0209f019/202402180000/resources, output/data/runs/b30a-0209f019/202402180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402180000.log - jobid: 509 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404140000/config.yaml, output/data/runs/b30a-0209f019/202404140000/resources, output/data/runs/b30a-0209f019/202404140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404140000.log - jobid: 677 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407070000/config.yaml, output/data/runs/b30a-0209f019/202407070000/resources, output/data/runs/b30a-0209f019/202407070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407070000.log - jobid: 929 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409150000.log - jobid: 260 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409290000/config.yaml, output/data/runs/b30a-0209f019/202409290000/resources, output/data/runs/b30a-0209f019/202409290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409290000.log - jobid: 1181 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405270000.log - jobid: 149 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409160000.log - jobid: 261 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408050000/config.yaml, output/data/runs/b30a-0209f019/202408050000/resources, output/data/runs/b30a-0209f019/202408050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408050000.log - jobid: 1016 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409020000/config.yaml, output/data/runs/b30a-0209f019/202409020000/resources, output/data/runs/b30a-0209f019/202409020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409020000.log - jobid: 1100 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401220000/config.yaml, output/data/runs/b30a-0209f019/202401220000/resources, output/data/runs/b30a-0209f019/202401220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401220000.log - jobid: 428 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405280000.log - jobid: 150 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402190000/config.yaml, output/data/runs/b30a-0209f019/202402190000/resources, output/data/runs/b30a-0209f019/202402190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402190000.log - jobid: 512 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404150000/config.yaml, output/data/runs/b30a-0209f019/202404150000/resources, output/data/runs/b30a-0209f019/202404150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404150000.log - jobid: 680 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407080000/config.yaml, output/data/runs/b30a-0209f019/202407080000/resources, output/data/runs/b30a-0209f019/202407080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407080000.log - jobid: 932 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409170000.log - jobid: 262 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409300000/config.yaml, output/data/runs/b30a-0209f019/202409300000/resources, output/data/runs/b30a-0209f019/202409300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409300000.log - jobid: 1184 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410280000/config.yaml, output/data/runs/b30a-0209f019/202410280000/resources, output/data/runs/b30a-0209f019/202410280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410280000.log - jobid: 1268 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405290000.log - jobid: 151 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409180000.log - jobid: 263 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401230000/config.yaml, output/data/runs/b30a-0209f019/202401230000/resources, output/data/runs/b30a-0209f019/202401230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401230000.log - jobid: 431 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408060000/config.yaml, output/data/runs/b30a-0209f019/202408060000/resources, output/data/runs/b30a-0209f019/202408060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408060000.log - jobid: 1019 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409030000/config.yaml, output/data/runs/b30a-0209f019/202409030000/resources, output/data/runs/b30a-0209f019/202409030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409030000.log - jobid: 1103 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405300000.log - jobid: 152 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402200000/config.yaml, output/data/runs/b30a-0209f019/202402200000/resources, output/data/runs/b30a-0209f019/202402200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402200000.log - jobid: 515 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404160000/config.yaml, output/data/runs/b30a-0209f019/202404160000/resources, output/data/runs/b30a-0209f019/202404160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404160000.log - jobid: 683 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407090000/config.yaml, output/data/runs/b30a-0209f019/202407090000/resources, output/data/runs/b30a-0209f019/202407090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407090000.log - jobid: 935 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409190000.log - jobid: 264 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410010000/config.yaml, output/data/runs/b30a-0209f019/202410010000/resources, output/data/runs/b30a-0209f019/202410010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410010000.log - jobid: 1187 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410290000/config.yaml, output/data/runs/b30a-0209f019/202410290000/resources, output/data/runs/b30a-0209f019/202410290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410290000.log - jobid: 1271 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405310000.log - jobid: 153 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409200000.log - jobid: 265 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401240000/config.yaml, output/data/runs/b30a-0209f019/202401240000/resources, output/data/runs/b30a-0209f019/202401240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401240000.log - jobid: 434 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408070000/config.yaml, output/data/runs/b30a-0209f019/202408070000/resources, output/data/runs/b30a-0209f019/202408070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408070000.log - jobid: 1022 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409040000/config.yaml, output/data/runs/b30a-0209f019/202409040000/resources, output/data/runs/b30a-0209f019/202409040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409040000.log - jobid: 1106 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406010000.log - jobid: 154 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402210000/config.yaml, output/data/runs/b30a-0209f019/202402210000/resources, output/data/runs/b30a-0209f019/202402210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402210000.log - jobid: 518 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407100000/config.yaml, output/data/runs/b30a-0209f019/202407100000/resources, output/data/runs/b30a-0209f019/202407100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407100000.log - jobid: 938 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404170000/config.yaml, output/data/runs/b30a-0209f019/202404170000/resources, output/data/runs/b30a-0209f019/202404170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404170000.log - jobid: 686 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409210000.log - jobid: 266 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410020000/config.yaml, output/data/runs/b30a-0209f019/202410020000/resources, output/data/runs/b30a-0209f019/202410020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410020000.log - jobid: 1190 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410300000/config.yaml, output/data/runs/b30a-0209f019/202410300000/resources, output/data/runs/b30a-0209f019/202410300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410300000.log - jobid: 1274 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406020000.log - jobid: 155 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409220000.log - jobid: 267 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401250000/config.yaml, output/data/runs/b30a-0209f019/202401250000/resources, output/data/runs/b30a-0209f019/202401250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401250000.log - jobid: 437 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408080000/config.yaml, output/data/runs/b30a-0209f019/202408080000/resources, output/data/runs/b30a-0209f019/202408080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408080000.log - jobid: 1025 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409050000/config.yaml, output/data/runs/b30a-0209f019/202409050000/resources, output/data/runs/b30a-0209f019/202409050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409050000.log - jobid: 1109 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406030000.log - jobid: 156 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402220000/config.yaml, output/data/runs/b30a-0209f019/202402220000/resources, output/data/runs/b30a-0209f019/202402220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402220000.log - jobid: 521 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404180000/config.yaml, output/data/runs/b30a-0209f019/202404180000/resources, output/data/runs/b30a-0209f019/202404180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404180000.log - jobid: 689 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407110000/config.yaml, output/data/runs/b30a-0209f019/202407110000/resources, output/data/runs/b30a-0209f019/202407110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407110000.log - jobid: 941 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:23 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409230000.log - jobid: 268 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410030000/config.yaml, output/data/runs/b30a-0209f019/202410030000/resources, output/data/runs/b30a-0209f019/202410030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410030000.log - jobid: 1193 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410310000/config.yaml, output/data/runs/b30a-0209f019/202410310000/resources, output/data/runs/b30a-0209f019/202410310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410310000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410310000.log - jobid: 1277 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406040000.log - jobid: 157 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409240000.log - jobid: 269 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401260000/config.yaml, output/data/runs/b30a-0209f019/202401260000/resources, output/data/runs/b30a-0209f019/202401260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401260000.log - jobid: 440 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408090000/config.yaml, output/data/runs/b30a-0209f019/202408090000/resources, output/data/runs/b30a-0209f019/202408090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408090000.log - jobid: 1028 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409060000/config.yaml, output/data/runs/b30a-0209f019/202409060000/resources, output/data/runs/b30a-0209f019/202409060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409060000.log - jobid: 1112 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406050000.log - jobid: 158 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402230000/config.yaml, output/data/runs/b30a-0209f019/202402230000/resources, output/data/runs/b30a-0209f019/202402230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402230000.log - jobid: 524 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404190000/config.yaml, output/data/runs/b30a-0209f019/202404190000/resources, output/data/runs/b30a-0209f019/202404190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404190000.log - jobid: 692 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407120000/config.yaml, output/data/runs/b30a-0209f019/202407120000/resources, output/data/runs/b30a-0209f019/202407120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407120000.log - jobid: 944 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409250000.log - jobid: 270 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410040000/config.yaml, output/data/runs/b30a-0209f019/202410040000/resources, output/data/runs/b30a-0209f019/202410040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410040000.log - jobid: 1196 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411010000/config.yaml, output/data/runs/b30a-0209f019/202411010000/resources, output/data/runs/b30a-0209f019/202411010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411010000.log - jobid: 1280 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406060000.log - jobid: 159 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409260000.log - jobid: 271 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401270000/config.yaml, output/data/runs/b30a-0209f019/202401270000/resources, output/data/runs/b30a-0209f019/202401270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401270000.log - jobid: 443 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409070000/config.yaml, output/data/runs/b30a-0209f019/202409070000/resources, output/data/runs/b30a-0209f019/202409070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409070000.log - jobid: 1115 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408100000/config.yaml, output/data/runs/b30a-0209f019/202408100000/resources, output/data/runs/b30a-0209f019/202408100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408100000.log - jobid: 1031 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406070000.log - jobid: 160 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402240000/config.yaml, output/data/runs/b30a-0209f019/202402240000/resources, output/data/runs/b30a-0209f019/202402240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402240000.log - jobid: 527 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404200000/config.yaml, output/data/runs/b30a-0209f019/202404200000/resources, output/data/runs/b30a-0209f019/202404200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404200000.log - jobid: 695 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407130000/config.yaml, output/data/runs/b30a-0209f019/202407130000/resources, output/data/runs/b30a-0209f019/202407130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407130000.log - jobid: 947 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409270000.log - jobid: 272 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410050000/config.yaml, output/data/runs/b30a-0209f019/202410050000/resources, output/data/runs/b30a-0209f019/202410050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410050000.log - jobid: 1199 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411020000/config.yaml, output/data/runs/b30a-0209f019/202411020000/resources, output/data/runs/b30a-0209f019/202411020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411020000.log - jobid: 1283 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406080000.log - jobid: 161 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409280000.log - jobid: 273 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401280000/config.yaml, output/data/runs/b30a-0209f019/202401280000/resources, output/data/runs/b30a-0209f019/202401280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401280000.log - jobid: 446 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409080000/config.yaml, output/data/runs/b30a-0209f019/202409080000/resources, output/data/runs/b30a-0209f019/202409080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409080000.log - jobid: 1118 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408110000/config.yaml, output/data/runs/b30a-0209f019/202408110000/resources, output/data/runs/b30a-0209f019/202408110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408110000.log - jobid: 1034 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406090000.log - jobid: 162 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402250000/config.yaml, output/data/runs/b30a-0209f019/202402250000/resources, output/data/runs/b30a-0209f019/202402250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402250000.log - jobid: 530 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404210000/config.yaml, output/data/runs/b30a-0209f019/202404210000/resources, output/data/runs/b30a-0209f019/202404210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404210000.log - jobid: 698 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407140000/config.yaml, output/data/runs/b30a-0209f019/202407140000/resources, output/data/runs/b30a-0209f019/202407140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407140000.log - jobid: 950 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule make_squashfs_image: - input: output/data/runs/b30a-0209f019/.venv - output: output/data/runs/b30a-0209f019/venv.squashfs - log: output/logs/make_squashfs_image/b30a-0209f019.log - jobid: 367 - reason: Missing output files: output/data/runs/b30a-0209f019/venv.squashfs; Updated input files: output/data/runs/b30a-0209f019/.venv - wildcards: run_id=b30a-0209f019 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409290000.log - jobid: 274 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410060000/config.yaml, output/data/runs/b30a-0209f019/202410060000/resources, output/data/runs/b30a-0209f019/202410060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410060000.log - jobid: 1202 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406100000.log - jobid: 163 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409300000.log - jobid: 275 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408120000/config.yaml, output/data/runs/b30a-0209f019/202408120000/resources, output/data/runs/b30a-0209f019/202408120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408120000.log - jobid: 1037 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401290000/config.yaml, output/data/runs/b30a-0209f019/202401290000/resources, output/data/runs/b30a-0209f019/202401290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401290000.log - jobid: 449 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409090000/config.yaml, output/data/runs/b30a-0209f019/202409090000/resources, output/data/runs/b30a-0209f019/202409090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409090000.log - jobid: 1121 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406110000.log - jobid: 164 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402260000/config.yaml, output/data/runs/b30a-0209f019/202402260000/resources, output/data/runs/b30a-0209f019/202402260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402260000.log - jobid: 533 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404220000/config.yaml, output/data/runs/b30a-0209f019/202404220000/resources, output/data/runs/b30a-0209f019/202404220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404220000.log - jobid: 701 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407150000/config.yaml, output/data/runs/b30a-0209f019/202407150000/resources, output/data/runs/b30a-0209f019/202407150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407150000.log - jobid: 953 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410010000.log - jobid: 276 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410070000/config.yaml, output/data/runs/b30a-0209f019/202410070000/resources, output/data/runs/b30a-0209f019/202410070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410070000.log - jobid: 1205 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411040000/config.yaml, output/data/runs/b30a-0209f019/202411040000/resources, output/data/runs/b30a-0209f019/202411040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411040000.log - jobid: 1289 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406120000.log - jobid: 165 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410020000.log - jobid: 277 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408130000/config.yaml, output/data/runs/b30a-0209f019/202408130000/resources, output/data/runs/b30a-0209f019/202408130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408130000.log - jobid: 1040 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401300000/config.yaml, output/data/runs/b30a-0209f019/202401300000/resources, output/data/runs/b30a-0209f019/202401300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401300000.log - jobid: 452 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409100000/config.yaml, output/data/runs/b30a-0209f019/202409100000/resources, output/data/runs/b30a-0209f019/202409100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409100000.log - jobid: 1124 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406130000.log - jobid: 166 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402270000/config.yaml, output/data/runs/b30a-0209f019/202402270000/resources, output/data/runs/b30a-0209f019/202402270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402270000.log - jobid: 536 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404230000/config.yaml, output/data/runs/b30a-0209f019/202404230000/resources, output/data/runs/b30a-0209f019/202404230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404230000.log - jobid: 704 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407160000/config.yaml, output/data/runs/b30a-0209f019/202407160000/resources, output/data/runs/b30a-0209f019/202407160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407160000.log - jobid: 956 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410030000.log - jobid: 278 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410080000/config.yaml, output/data/runs/b30a-0209f019/202410080000/resources, output/data/runs/b30a-0209f019/202410080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410080000.log - jobid: 1208 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411050000/config.yaml, output/data/runs/b30a-0209f019/202411050000/resources, output/data/runs/b30a-0209f019/202411050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411050000.log - jobid: 1292 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406140000.log - jobid: 167 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410040000.log - jobid: 279 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402280000/config.yaml, output/data/runs/b30a-0209f019/202402280000/resources, output/data/runs/b30a-0209f019/202402280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402280000.log - jobid: 539 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404240000/config.yaml, output/data/runs/b30a-0209f019/202404240000/resources, output/data/runs/b30a-0209f019/202404240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404240000.log - jobid: 707 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407170000/config.yaml, output/data/runs/b30a-0209f019/202407170000/resources, output/data/runs/b30a-0209f019/202407170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407170000.log - jobid: 959 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401310000/config.yaml, output/data/runs/b30a-0209f019/202401310000/resources, output/data/runs/b30a-0209f019/202401310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401310000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401310000.log - jobid: 455 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406150000.log - jobid: 168 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401030000/config.yaml, output/data/runs/b30a-0209f019/202401030000/resources, output/data/runs/b30a-0209f019/202401030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401030000.log - jobid: 371 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409110000/config.yaml, output/data/runs/b30a-0209f019/202409110000/resources, output/data/runs/b30a-0209f019/202409110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409110000.log - jobid: 1127 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411060000/config.yaml, output/data/runs/b30a-0209f019/202411060000/resources, output/data/runs/b30a-0209f019/202411060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411060000.log - jobid: 1295 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410050000.log - jobid: 280 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408140000/config.yaml, output/data/runs/b30a-0209f019/202408140000/resources, output/data/runs/b30a-0209f019/202408140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408140000.log - jobid: 1043 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406160000.log - jobid: 169 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410060000.log - jobid: 281 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402010000/config.yaml, output/data/runs/b30a-0209f019/202402010000/resources, output/data/runs/b30a-0209f019/202402010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402010000.log - jobid: 458 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408150000/config.yaml, output/data/runs/b30a-0209f019/202408150000/resources, output/data/runs/b30a-0209f019/202408150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408150000.log - jobid: 1046 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409120000/config.yaml, output/data/runs/b30a-0209f019/202409120000/resources, output/data/runs/b30a-0209f019/202409120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409120000.log - jobid: 1130 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406170000.log - jobid: 170 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402290000/config.yaml, output/data/runs/b30a-0209f019/202402290000/resources, output/data/runs/b30a-0209f019/202402290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402290000.log - jobid: 542 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401040000/config.yaml, output/data/runs/b30a-0209f019/202401040000/resources, output/data/runs/b30a-0209f019/202401040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401040000.log - jobid: 374 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404250000/config.yaml, output/data/runs/b30a-0209f019/202404250000/resources, output/data/runs/b30a-0209f019/202404250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404250000.log - jobid: 710 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407180000/config.yaml, output/data/runs/b30a-0209f019/202407180000/resources, output/data/runs/b30a-0209f019/202407180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407180000.log - jobid: 962 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410070000.log - jobid: 282 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411070000/config.yaml, output/data/runs/b30a-0209f019/202411070000/resources, output/data/runs/b30a-0209f019/202411070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411070000.log - jobid: 1298 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401020000.log - jobid: 3 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406180000.log - jobid: 171 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410080000.log - jobid: 283 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409130000/config.yaml, output/data/runs/b30a-0209f019/202409130000/resources, output/data/runs/b30a-0209f019/202409130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409130000.log - jobid: 1133 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402020000/config.yaml, output/data/runs/b30a-0209f019/202402020000/resources, output/data/runs/b30a-0209f019/202402020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402020000.log - jobid: 461 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408160000/config.yaml, output/data/runs/b30a-0209f019/202408160000/resources, output/data/runs/b30a-0209f019/202408160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408160000.log - jobid: 1049 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406190000.log - jobid: 172 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401030000.log - jobid: 4 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403010000/config.yaml, output/data/runs/b30a-0209f019/202403010000/resources, output/data/runs/b30a-0209f019/202403010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403010000.log - jobid: 545 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401050000/config.yaml, output/data/runs/b30a-0209f019/202401050000/resources, output/data/runs/b30a-0209f019/202401050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401050000.log - jobid: 377 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404260000/config.yaml, output/data/runs/b30a-0209f019/202404260000/resources, output/data/runs/b30a-0209f019/202404260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404260000.log - jobid: 713 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410090000.log - jobid: 284 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411080000/config.yaml, output/data/runs/b30a-0209f019/202411080000/resources, output/data/runs/b30a-0209f019/202411080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411080000.log - jobid: 1301 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406200000.log - jobid: 173 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401040000.log - jobid: 5 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410100000.log - jobid: 285 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403020000/config.yaml, output/data/runs/b30a-0209f019/202403020000/resources, output/data/runs/b30a-0209f019/202403020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403020000.log - jobid: 548 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409140000/config.yaml, output/data/runs/b30a-0209f019/202409140000/resources, output/data/runs/b30a-0209f019/202409140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409140000.log - jobid: 1136 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402030000/config.yaml, output/data/runs/b30a-0209f019/202402030000/resources, output/data/runs/b30a-0209f019/202402030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402030000.log - jobid: 464 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408170000/config.yaml, output/data/runs/b30a-0209f019/202408170000/resources, output/data/runs/b30a-0209f019/202408170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408170000.log - jobid: 1052 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406210000.log - jobid: 174 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401050000.log - jobid: 6 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404270000/config.yaml, output/data/runs/b30a-0209f019/202404270000/resources, output/data/runs/b30a-0209f019/202404270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404270000.log - jobid: 716 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407200000/config.yaml, output/data/runs/b30a-0209f019/202407200000/resources, output/data/runs/b30a-0209f019/202407200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407200000.log - jobid: 968 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401060000/config.yaml, output/data/runs/b30a-0209f019/202401060000/resources, output/data/runs/b30a-0209f019/202401060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401060000.log - jobid: 380 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410110000.log - jobid: 286 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406220000.log - jobid: 175 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401060000.log - jobid: 7 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410120000.log - jobid: 287 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409150000/config.yaml, output/data/runs/b30a-0209f019/202409150000/resources, output/data/runs/b30a-0209f019/202409150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409150000.log - jobid: 1139 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402040000/config.yaml, output/data/runs/b30a-0209f019/202402040000/resources, output/data/runs/b30a-0209f019/202402040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402040000.log - jobid: 467 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408180000/config.yaml, output/data/runs/b30a-0209f019/202408180000/resources, output/data/runs/b30a-0209f019/202408180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408180000.log - jobid: 1055 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401070000.log - jobid: 8 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406230000.log - jobid: 176 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403030000/config.yaml, output/data/runs/b30a-0209f019/202403030000/resources, output/data/runs/b30a-0209f019/202403030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403030000.log - jobid: 551 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404280000/config.yaml, output/data/runs/b30a-0209f019/202404280000/resources, output/data/runs/b30a-0209f019/202404280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404280000.log - jobid: 719 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407210000/config.yaml, output/data/runs/b30a-0209f019/202407210000/resources, output/data/runs/b30a-0209f019/202407210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407210000.log - jobid: 971 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410130000.log - jobid: 288 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410130000/config.yaml, output/data/runs/b30a-0209f019/202410130000/resources, output/data/runs/b30a-0209f019/202410130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410130000.log - jobid: 1223 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401080000.log - jobid: 9 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406240000.log - jobid: 177 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401070000/config.yaml, output/data/runs/b30a-0209f019/202401070000/resources, output/data/runs/b30a-0209f019/202401070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401070000.log - jobid: 383 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410140000.log - jobid: 289 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410270000/config.yaml, output/data/runs/b30a-0209f019/202410270000/resources, output/data/runs/b30a-0209f019/202410270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410270000.log - jobid: 1265 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409160000/config.yaml, output/data/runs/b30a-0209f019/202409160000/resources, output/data/runs/b30a-0209f019/202409160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409160000.log - jobid: 1142 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402050000/config.yaml, output/data/runs/b30a-0209f019/202402050000/resources, output/data/runs/b30a-0209f019/202402050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402050000.log - jobid: 470 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408190000/config.yaml, output/data/runs/b30a-0209f019/202408190000/resources, output/data/runs/b30a-0209f019/202408190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408190000.log - jobid: 1058 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401090000.log - jobid: 10 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406250000.log - jobid: 178 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403040000/config.yaml, output/data/runs/b30a-0209f019/202403040000/resources, output/data/runs/b30a-0209f019/202403040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403040000.log - jobid: 554 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404290000/config.yaml, output/data/runs/b30a-0209f019/202404290000/resources, output/data/runs/b30a-0209f019/202404290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404290000.log - jobid: 722 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407220000/config.yaml, output/data/runs/b30a-0209f019/202407220000/resources, output/data/runs/b30a-0209f019/202407220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407220000.log - jobid: 974 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410150000.log - jobid: 290 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410140000/config.yaml, output/data/runs/b30a-0209f019/202410140000/resources, output/data/runs/b30a-0209f019/202410140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410140000.log - jobid: 1226 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401100000.log - jobid: 11 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406260000.log - jobid: 179 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410160000.log - jobid: 291 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410170000.log - jobid: 292 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403050000/config.yaml, output/data/runs/b30a-0209f019/202403050000/resources, output/data/runs/b30a-0209f019/202403050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403050000.log - jobid: 557 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402060000/config.yaml, output/data/runs/b30a-0209f019/202402060000/resources, output/data/runs/b30a-0209f019/202402060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402060000.log - jobid: 473 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409170000/config.yaml, output/data/runs/b30a-0209f019/202409170000/resources, output/data/runs/b30a-0209f019/202409170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409170000.log - jobid: 1145 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408200000/config.yaml, output/data/runs/b30a-0209f019/202408200000/resources, output/data/runs/b30a-0209f019/202408200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408200000.log - jobid: 1061 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401110000.log - jobid: 12 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406270000.log - jobid: 180 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404300000/config.yaml, output/data/runs/b30a-0209f019/202404300000/resources, output/data/runs/b30a-0209f019/202404300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404300000.log - jobid: 725 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407230000/config.yaml, output/data/runs/b30a-0209f019/202407230000/resources, output/data/runs/b30a-0209f019/202407230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407230000.log - jobid: 977 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410150000/config.yaml, output/data/runs/b30a-0209f019/202410150000/resources, output/data/runs/b30a-0209f019/202410150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410150000.log - jobid: 1229 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401120000.log - jobid: 13 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406280000.log - jobid: 181 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401080000/config.yaml, output/data/runs/b30a-0209f019/202401080000/resources, output/data/runs/b30a-0209f019/202401080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401080000.log - jobid: 386 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410180000.log - jobid: 293 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410190000.log - jobid: 294 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408210000/config.yaml, output/data/runs/b30a-0209f019/202408210000/resources, output/data/runs/b30a-0209f019/202408210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408210000.log - jobid: 1064 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402070000/config.yaml, output/data/runs/b30a-0209f019/202402070000/resources, output/data/runs/b30a-0209f019/202402070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402070000.log - jobid: 476 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409180000/config.yaml, output/data/runs/b30a-0209f019/202409180000/resources, output/data/runs/b30a-0209f019/202409180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409180000.log - jobid: 1148 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401130000.log - jobid: 14 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406290000.log - jobid: 182 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403060000/config.yaml, output/data/runs/b30a-0209f019/202403060000/resources, output/data/runs/b30a-0209f019/202403060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403060000.log - jobid: 560 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407240000/config.yaml, output/data/runs/b30a-0209f019/202407240000/resources, output/data/runs/b30a-0209f019/202407240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407240000.log - jobid: 980 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405010000/config.yaml, output/data/runs/b30a-0209f019/202405010000/resources, output/data/runs/b30a-0209f019/202405010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405010000.log - jobid: 728 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410160000/config.yaml, output/data/runs/b30a-0209f019/202410160000/resources, output/data/runs/b30a-0209f019/202410160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410160000.log - jobid: 1232 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401140000.log - jobid: 15 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406300000.log - jobid: 183 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202406300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401090000/config.yaml, output/data/runs/b30a-0209f019/202401090000/resources, output/data/runs/b30a-0209f019/202401090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401090000.log - jobid: 389 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410200000.log - jobid: 295 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407010000.log - jobid: 184 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402080000/config.yaml, output/data/runs/b30a-0209f019/202402080000/resources, output/data/runs/b30a-0209f019/202402080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402080000.log - jobid: 479 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409190000/config.yaml, output/data/runs/b30a-0209f019/202409190000/resources, output/data/runs/b30a-0209f019/202409190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409190000.log - jobid: 1151 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408220000/config.yaml, output/data/runs/b30a-0209f019/202408220000/resources, output/data/runs/b30a-0209f019/202408220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408220000.log - jobid: 1067 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401150000.log - jobid: 16 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403070000/config.yaml, output/data/runs/b30a-0209f019/202403070000/resources, output/data/runs/b30a-0209f019/202403070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403070000.log - jobid: 563 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405020000/config.yaml, output/data/runs/b30a-0209f019/202405020000/resources, output/data/runs/b30a-0209f019/202405020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405020000.log - jobid: 731 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407250000/config.yaml, output/data/runs/b30a-0209f019/202407250000/resources, output/data/runs/b30a-0209f019/202407250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407250000.log - jobid: 983 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410170000/config.yaml, output/data/runs/b30a-0209f019/202410170000/resources, output/data/runs/b30a-0209f019/202410170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410170000.log - jobid: 1235 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410210000.log - jobid: 296 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401160000.log - jobid: 17 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407020000.log - jobid: 185 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401100000/config.yaml, output/data/runs/b30a-0209f019/202401100000/resources, output/data/runs/b30a-0209f019/202401100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401100000.log - jobid: 392 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410220000.log - jobid: 297 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411150000/config.yaml, output/data/runs/b30a-0209f019/202411150000/resources, output/data/runs/b30a-0209f019/202411150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411150000.log - jobid: 1322 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407260000/config.yaml, output/data/runs/b30a-0209f019/202407260000/resources, output/data/runs/b30a-0209f019/202407260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407260000.log - jobid: 986 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410180000/config.yaml, output/data/runs/b30a-0209f019/202410180000/resources, output/data/runs/b30a-0209f019/202410180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410180000.log - jobid: 1238 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403080000/config.yaml, output/data/runs/b30a-0209f019/202403080000/resources, output/data/runs/b30a-0209f019/202403080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403080000.log - jobid: 566 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408230000/config.yaml, output/data/runs/b30a-0209f019/202408230000/resources, output/data/runs/b30a-0209f019/202408230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408230000.log - jobid: 1070 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402090000/config.yaml, output/data/runs/b30a-0209f019/202402090000/resources, output/data/runs/b30a-0209f019/202402090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402090000.log - jobid: 482 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409200000/config.yaml, output/data/runs/b30a-0209f019/202409200000/resources, output/data/runs/b30a-0209f019/202409200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409200000.log - jobid: 1154 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401170000.log - jobid: 18 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407030000.log - jobid: 186 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405030000/config.yaml, output/data/runs/b30a-0209f019/202405030000/resources, output/data/runs/b30a-0209f019/202405030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405030000.log - jobid: 734 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401180000.log - jobid: 19 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407040000.log - jobid: 187 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401110000/config.yaml, output/data/runs/b30a-0209f019/202401110000/resources, output/data/runs/b30a-0209f019/202401110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401110000.log - jobid: 395 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410240000.log - jobid: 299 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403090000/config.yaml, output/data/runs/b30a-0209f019/202403090000/resources, output/data/runs/b30a-0209f019/202403090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403090000.log - jobid: 569 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408240000/config.yaml, output/data/runs/b30a-0209f019/202408240000/resources, output/data/runs/b30a-0209f019/202408240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408240000.log - jobid: 1073 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402100000/config.yaml, output/data/runs/b30a-0209f019/202402100000/resources, output/data/runs/b30a-0209f019/202402100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402100000.log - jobid: 485 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409210000/config.yaml, output/data/runs/b30a-0209f019/202409210000/resources, output/data/runs/b30a-0209f019/202409210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409210000.log - jobid: 1157 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401190000.log - jobid: 20 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407050000.log - jobid: 188 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401120000/config.yaml, output/data/runs/b30a-0209f019/202401120000/resources, output/data/runs/b30a-0209f019/202401120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401120000.log - jobid: 398 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405040000/config.yaml, output/data/runs/b30a-0209f019/202405040000/resources, output/data/runs/b30a-0209f019/202405040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405040000.log - jobid: 737 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407270000/config.yaml, output/data/runs/b30a-0209f019/202407270000/resources, output/data/runs/b30a-0209f019/202407270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407270000.log - jobid: 989 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411160000/config.yaml, output/data/runs/b30a-0209f019/202411160000/resources, output/data/runs/b30a-0209f019/202411160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411160000.log - jobid: 1325 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401200000.log - jobid: 21 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407060000.log - jobid: 189 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410260000.log - jobid: 301 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403100000/config.yaml, output/data/runs/b30a-0209f019/202403100000/resources, output/data/runs/b30a-0209f019/202403100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403100000.log - jobid: 572 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408250000/config.yaml, output/data/runs/b30a-0209f019/202408250000/resources, output/data/runs/b30a-0209f019/202408250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408250000.log - jobid: 1076 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402110000/config.yaml, output/data/runs/b30a-0209f019/202402110000/resources, output/data/runs/b30a-0209f019/202402110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402110000.log - jobid: 488 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409220000/config.yaml, output/data/runs/b30a-0209f019/202409220000/resources, output/data/runs/b30a-0209f019/202409220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409220000.log - jobid: 1160 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401210000.log - jobid: 22 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407070000.log - jobid: 190 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407280000/config.yaml, output/data/runs/b30a-0209f019/202407280000/resources, output/data/runs/b30a-0209f019/202407280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407280000.log - jobid: 992 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401140000/config.yaml, output/data/runs/b30a-0209f019/202401140000/resources, output/data/runs/b30a-0209f019/202401140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401140000.log - jobid: 404 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405050000/config.yaml, output/data/runs/b30a-0209f019/202405050000/resources, output/data/runs/b30a-0209f019/202405050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405050000.log - jobid: 740 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411170000/config.yaml, output/data/runs/b30a-0209f019/202411170000/resources, output/data/runs/b30a-0209f019/202411170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411170000.log - jobid: 1328 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401220000.log - jobid: 23 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407080000.log - jobid: 191 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410280000.log - jobid: 303 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403110000/config.yaml, output/data/runs/b30a-0209f019/202403110000/resources, output/data/runs/b30a-0209f019/202403110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403110000.log - jobid: 575 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408260000/config.yaml, output/data/runs/b30a-0209f019/202408260000/resources, output/data/runs/b30a-0209f019/202408260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408260000.log - jobid: 1079 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402120000/config.yaml, output/data/runs/b30a-0209f019/202402120000/resources, output/data/runs/b30a-0209f019/202402120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402120000.log - jobid: 491 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409230000/config.yaml, output/data/runs/b30a-0209f019/202409230000/resources, output/data/runs/b30a-0209f019/202409230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409230000.log - jobid: 1163 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401230000.log - jobid: 24 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407090000.log - jobid: 192 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405060000/config.yaml, output/data/runs/b30a-0209f019/202405060000/resources, output/data/runs/b30a-0209f019/202405060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405060000.log - jobid: 743 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407290000/config.yaml, output/data/runs/b30a-0209f019/202407290000/resources, output/data/runs/b30a-0209f019/202407290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407290000.log - jobid: 995 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401150000/config.yaml, output/data/runs/b30a-0209f019/202401150000/resources, output/data/runs/b30a-0209f019/202401150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401150000.log - jobid: 407 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411180000/config.yaml, output/data/runs/b30a-0209f019/202411180000/resources, output/data/runs/b30a-0209f019/202411180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411180000.log - jobid: 1331 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401240000.log - jobid: 25 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407100000.log - jobid: 193 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410300000.log - jobid: 305 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411190000/config.yaml, output/data/runs/b30a-0209f019/202411190000/resources, output/data/runs/b30a-0209f019/202411190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411190000.log - jobid: 1334 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407110000.log - jobid: 194 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202409240000/config.yaml, output/data/runs/b30a-0209f019/202409240000/resources, output/data/runs/b30a-0209f019/202409240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409240000.log - jobid: 1166 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202409240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202408270000/config.yaml, output/data/runs/b30a-0209f019/202408270000/resources, output/data/runs/b30a-0209f019/202408270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408270000.log - jobid: 1082 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202408270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202402130000/config.yaml, output/data/runs/b30a-0209f019/202402130000/resources, output/data/runs/b30a-0209f019/202402130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402130000.log - jobid: 494 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202402130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403120000/config.yaml, output/data/runs/b30a-0209f019/202403120000/resources, output/data/runs/b30a-0209f019/202403120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403120000.log - jobid: 578 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401250000.log - jobid: 26 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405070000/config.yaml, output/data/runs/b30a-0209f019/202405070000/resources, output/data/runs/b30a-0209f019/202405070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405070000.log - jobid: 746 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407300000/config.yaml, output/data/runs/b30a-0209f019/202407300000/resources, output/data/runs/b30a-0209f019/202407300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407300000.log - jobid: 998 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401160000/config.yaml, output/data/runs/b30a-0209f019/202401160000/resources, output/data/runs/b30a-0209f019/202401160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401160000.log - jobid: 410 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410190000/config.yaml, output/data/runs/b30a-0209f019/202410190000/resources, output/data/runs/b30a-0209f019/202410190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410190000.log - jobid: 1241 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411090000/config.yaml, output/data/runs/b30a-0209f019/202411090000/resources, output/data/runs/b30a-0209f019/202411090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411090000.log - jobid: 1304 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401260000.log - jobid: 27 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407120000.log - jobid: 195 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411010000.log - jobid: 307 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403130000/config.yaml, output/data/runs/b30a-0209f019/202403130000/resources, output/data/runs/b30a-0209f019/202403130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403130000.log - jobid: 581 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405080000/config.yaml, output/data/runs/b30a-0209f019/202405080000/resources, output/data/runs/b30a-0209f019/202405080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405080000.log - jobid: 749 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411200000/config.yaml, output/data/runs/b30a-0209f019/202411200000/resources, output/data/runs/b30a-0209f019/202411200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411200000.log - jobid: 1337 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407130000.log - jobid: 196 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406050000/config.yaml, output/data/runs/b30a-0209f019/202406050000/resources, output/data/runs/b30a-0209f019/202406050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406050000.log - jobid: 833 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412180000/config.yaml, output/data/runs/b30a-0209f019/202412180000/resources, output/data/runs/b30a-0209f019/202412180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412180000.log - jobid: 1421 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401270000.log - jobid: 28 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403230000.log - jobid: 84 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411020000.log - jobid: 308 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407140000.log - jobid: 197 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401280000.log - jobid: 29 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403240000.log - jobid: 85 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411030000.log - jobid: 309 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410230000.log - jobid: 298 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403140000/config.yaml, output/data/runs/b30a-0209f019/202403140000/resources, output/data/runs/b30a-0209f019/202403140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403140000.log - jobid: 584 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405090000/config.yaml, output/data/runs/b30a-0209f019/202405090000/resources, output/data/runs/b30a-0209f019/202405090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405090000.log - jobid: 752 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411210000/config.yaml, output/data/runs/b30a-0209f019/202411210000/resources, output/data/runs/b30a-0209f019/202411210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411210000.log - jobid: 1340 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407150000.log - jobid: 198 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406060000/config.yaml, output/data/runs/b30a-0209f019/202406060000/resources, output/data/runs/b30a-0209f019/202406060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406060000.log - jobid: 836 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412190000/config.yaml, output/data/runs/b30a-0209f019/202412190000/resources, output/data/runs/b30a-0209f019/202412190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412190000.log - jobid: 1424 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401290000.log - jobid: 30 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403250000.log - jobid: 86 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411040000.log - jobid: 310 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407160000.log - jobid: 199 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401300000.log - jobid: 31 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403260000.log - jobid: 87 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411050000.log - jobid: 311 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403150000/config.yaml, output/data/runs/b30a-0209f019/202403150000/resources, output/data/runs/b30a-0209f019/202403150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403150000.log - jobid: 587 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405100000/config.yaml, output/data/runs/b30a-0209f019/202405100000/resources, output/data/runs/b30a-0209f019/202405100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405100000.log - jobid: 755 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411220000/config.yaml, output/data/runs/b30a-0209f019/202411220000/resources, output/data/runs/b30a-0209f019/202411220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411220000.log - jobid: 1343 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407170000.log - jobid: 200 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406070000/config.yaml, output/data/runs/b30a-0209f019/202406070000/resources, output/data/runs/b30a-0209f019/202406070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406070000.log - jobid: 839 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412200000/config.yaml, output/data/runs/b30a-0209f019/202412200000/resources, output/data/runs/b30a-0209f019/202412200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412200000.log - jobid: 1427 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401310000.log - jobid: 32 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202401310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403270000.log - jobid: 88 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411060000.log - jobid: 312 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407180000.log - jobid: 201 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402010000.log - jobid: 33 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403280000.log - jobid: 89 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411070000.log - jobid: 313 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403160000/config.yaml, output/data/runs/b30a-0209f019/202403160000/resources, output/data/runs/b30a-0209f019/202403160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403160000.log - jobid: 590 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405110000/config.yaml, output/data/runs/b30a-0209f019/202405110000/resources, output/data/runs/b30a-0209f019/202405110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405110000.log - jobid: 758 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411230000/config.yaml, output/data/runs/b30a-0209f019/202411230000/resources, output/data/runs/b30a-0209f019/202411230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411230000.log - jobid: 1346 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407190000.log - jobid: 202 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406080000/config.yaml, output/data/runs/b30a-0209f019/202406080000/resources, output/data/runs/b30a-0209f019/202406080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406080000.log - jobid: 842 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412210000/config.yaml, output/data/runs/b30a-0209f019/202412210000/resources, output/data/runs/b30a-0209f019/202412210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412210000.log - jobid: 1430 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402020000.log - jobid: 34 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403290000.log - jobid: 90 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411080000.log - jobid: 314 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407200000.log - jobid: 203 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402030000.log - jobid: 35 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403300000.log - jobid: 91 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411090000.log - jobid: 315 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410200000/config.yaml, output/data/runs/b30a-0209f019/202410200000/resources, output/data/runs/b30a-0209f019/202410200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410200000.log - jobid: 1244 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403170000/config.yaml, output/data/runs/b30a-0209f019/202403170000/resources, output/data/runs/b30a-0209f019/202403170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403170000.log - jobid: 593 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405120000/config.yaml, output/data/runs/b30a-0209f019/202405120000/resources, output/data/runs/b30a-0209f019/202405120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405120000.log - jobid: 761 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411240000/config.yaml, output/data/runs/b30a-0209f019/202411240000/resources, output/data/runs/b30a-0209f019/202411240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411240000.log - jobid: 1349 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407210000.log - jobid: 204 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406090000/config.yaml, output/data/runs/b30a-0209f019/202406090000/resources, output/data/runs/b30a-0209f019/202406090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406090000.log - jobid: 845 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412220000/config.yaml, output/data/runs/b30a-0209f019/202412220000/resources, output/data/runs/b30a-0209f019/202412220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412220000.log - jobid: 1433 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411100000/config.yaml, output/data/runs/b30a-0209f019/202411100000/resources, output/data/runs/b30a-0209f019/202411100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411100000.log - jobid: 1307 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402040000.log - jobid: 36 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403310000.log - jobid: 92 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411100000.log - jobid: 316 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407220000.log - jobid: 205 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402050000.log - jobid: 37 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404010000.log - jobid: 93 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411110000.log - jobid: 317 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403180000/config.yaml, output/data/runs/b30a-0209f019/202403180000/resources, output/data/runs/b30a-0209f019/202403180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403180000.log - jobid: 596 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405130000/config.yaml, output/data/runs/b30a-0209f019/202405130000/resources, output/data/runs/b30a-0209f019/202405130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405130000.log - jobid: 764 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411250000/config.yaml, output/data/runs/b30a-0209f019/202411250000/resources, output/data/runs/b30a-0209f019/202411250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411250000.log - jobid: 1352 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407230000.log - jobid: 206 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406100000/config.yaml, output/data/runs/b30a-0209f019/202406100000/resources, output/data/runs/b30a-0209f019/202406100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406100000.log - jobid: 848 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412230000/config.yaml, output/data/runs/b30a-0209f019/202412230000/resources, output/data/runs/b30a-0209f019/202412230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412230000.log - jobid: 1436 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402060000.log - jobid: 38 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404020000.log - jobid: 94 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410090000/config.yaml, output/data/runs/b30a-0209f019/202410090000/resources, output/data/runs/b30a-0209f019/202410090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410090000.log - jobid: 1211 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411120000.log - jobid: 318 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407240000.log - jobid: 207 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402070000.log - jobid: 39 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404030000.log - jobid: 95 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411130000.log - jobid: 319 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410250000.log - jobid: 300 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403190000/config.yaml, output/data/runs/b30a-0209f019/202403190000/resources, output/data/runs/b30a-0209f019/202403190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403190000.log - jobid: 599 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405140000/config.yaml, output/data/runs/b30a-0209f019/202405140000/resources, output/data/runs/b30a-0209f019/202405140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405140000.log - jobid: 767 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411260000/config.yaml, output/data/runs/b30a-0209f019/202411260000/resources, output/data/runs/b30a-0209f019/202411260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411260000.log - jobid: 1355 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407250000.log - jobid: 208 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406110000/config.yaml, output/data/runs/b30a-0209f019/202406110000/resources, output/data/runs/b30a-0209f019/202406110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406110000.log - jobid: 851 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412240000/config.yaml, output/data/runs/b30a-0209f019/202412240000/resources, output/data/runs/b30a-0209f019/202412240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412240000.log - jobid: 1439 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402080000.log - jobid: 40 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404040000.log - jobid: 96 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411140000.log - jobid: 320 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407260000.log - jobid: 209 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402090000.log - jobid: 41 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404050000.log - jobid: 97 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411150000.log - jobid: 321 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403200000/config.yaml, output/data/runs/b30a-0209f019/202403200000/resources, output/data/runs/b30a-0209f019/202403200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403200000.log - jobid: 602 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405150000/config.yaml, output/data/runs/b30a-0209f019/202405150000/resources, output/data/runs/b30a-0209f019/202405150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405150000.log - jobid: 770 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411270000/config.yaml, output/data/runs/b30a-0209f019/202411270000/resources, output/data/runs/b30a-0209f019/202411270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411270000.log - jobid: 1358 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406120000/config.yaml, output/data/runs/b30a-0209f019/202406120000/resources, output/data/runs/b30a-0209f019/202406120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406120000.log - jobid: 854 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407270000.log - jobid: 210 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412250000/config.yaml, output/data/runs/b30a-0209f019/202412250000/resources, output/data/runs/b30a-0209f019/202412250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412250000.log - jobid: 1442 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402100000.log - jobid: 42 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404060000.log - jobid: 98 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411160000.log - jobid: 322 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407280000.log - jobid: 211 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402110000.log - jobid: 43 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404070000.log - jobid: 99 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411170000.log - jobid: 323 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403210000/config.yaml, output/data/runs/b30a-0209f019/202403210000/resources, output/data/runs/b30a-0209f019/202403210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403210000.log - jobid: 605 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405160000/config.yaml, output/data/runs/b30a-0209f019/202405160000/resources, output/data/runs/b30a-0209f019/202405160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405160000.log - jobid: 773 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411280000/config.yaml, output/data/runs/b30a-0209f019/202411280000/resources, output/data/runs/b30a-0209f019/202411280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411280000.log - jobid: 1361 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407290000.log - jobid: 212 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406130000/config.yaml, output/data/runs/b30a-0209f019/202406130000/resources, output/data/runs/b30a-0209f019/202406130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406130000.log - jobid: 857 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402120000.log - jobid: 44 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404080000.log - jobid: 100 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411180000.log - jobid: 324 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407300000.log - jobid: 213 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402130000.log - jobid: 45 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404090000.log - jobid: 101 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411190000.log - jobid: 325 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410210000/config.yaml, output/data/runs/b30a-0209f019/202410210000/resources, output/data/runs/b30a-0209f019/202410210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410210000.log - jobid: 1247 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403220000/config.yaml, output/data/runs/b30a-0209f019/202403220000/resources, output/data/runs/b30a-0209f019/202403220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403220000.log - jobid: 608 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405170000/config.yaml, output/data/runs/b30a-0209f019/202405170000/resources, output/data/runs/b30a-0209f019/202405170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405170000.log - jobid: 776 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411290000/config.yaml, output/data/runs/b30a-0209f019/202411290000/resources, output/data/runs/b30a-0209f019/202411290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411290000.log - jobid: 1364 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407310000.log - jobid: 214 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202407310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406140000/config.yaml, output/data/runs/b30a-0209f019/202406140000/resources, output/data/runs/b30a-0209f019/202406140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406140000.log - jobid: 860 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411110000/config.yaml, output/data/runs/b30a-0209f019/202411110000/resources, output/data/runs/b30a-0209f019/202411110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411110000.log - jobid: 1310 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402140000.log - jobid: 46 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404100000.log - jobid: 102 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411200000.log - jobid: 326 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408010000.log - jobid: 215 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402150000.log - jobid: 47 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404110000.log - jobid: 103 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411210000.log - jobid: 327 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403230000/config.yaml, output/data/runs/b30a-0209f019/202403230000/resources, output/data/runs/b30a-0209f019/202403230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403230000.log - jobid: 611 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405180000/config.yaml, output/data/runs/b30a-0209f019/202405180000/resources, output/data/runs/b30a-0209f019/202405180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405180000.log - jobid: 779 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411300000/config.yaml, output/data/runs/b30a-0209f019/202411300000/resources, output/data/runs/b30a-0209f019/202411300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411300000.log - jobid: 1367 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408020000.log - jobid: 216 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406150000/config.yaml, output/data/runs/b30a-0209f019/202406150000/resources, output/data/runs/b30a-0209f019/202406150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406150000.log - jobid: 863 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402160000.log - jobid: 48 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404120000.log - jobid: 104 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411220000.log - jobid: 328 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410100000/config.yaml, output/data/runs/b30a-0209f019/202410100000/resources, output/data/runs/b30a-0209f019/202410100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410100000.log - jobid: 1214 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408030000.log - jobid: 217 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402170000.log - jobid: 49 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404130000.log - jobid: 105 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411230000.log - jobid: 329 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410270000.log - jobid: 302 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403240000/config.yaml, output/data/runs/b30a-0209f019/202403240000/resources, output/data/runs/b30a-0209f019/202403240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403240000.log - jobid: 614 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405190000/config.yaml, output/data/runs/b30a-0209f019/202405190000/resources, output/data/runs/b30a-0209f019/202405190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405190000.log - jobid: 782 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412010000/config.yaml, output/data/runs/b30a-0209f019/202412010000/resources, output/data/runs/b30a-0209f019/202412010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412010000.log - jobid: 1370 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408040000.log - jobid: 218 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406160000/config.yaml, output/data/runs/b30a-0209f019/202406160000/resources, output/data/runs/b30a-0209f019/202406160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406160000.log - jobid: 866 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402180000.log - jobid: 50 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404140000.log - jobid: 106 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411240000.log - jobid: 330 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408050000.log - jobid: 219 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402190000.log - jobid: 51 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404150000.log - jobid: 107 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411250000.log - jobid: 331 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403250000/config.yaml, output/data/runs/b30a-0209f019/202403250000/resources, output/data/runs/b30a-0209f019/202403250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403250000.log - jobid: 617 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405200000/config.yaml, output/data/runs/b30a-0209f019/202405200000/resources, output/data/runs/b30a-0209f019/202405200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405200000.log - jobid: 785 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412020000/config.yaml, output/data/runs/b30a-0209f019/202412020000/resources, output/data/runs/b30a-0209f019/202412020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412020000.log - jobid: 1373 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408060000.log - jobid: 220 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406170000/config.yaml, output/data/runs/b30a-0209f019/202406170000/resources, output/data/runs/b30a-0209f019/202406170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406170000.log - jobid: 869 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402200000.log - jobid: 52 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404160000.log - jobid: 108 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411260000.log - jobid: 332 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408070000.log - jobid: 221 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402210000.log - jobid: 53 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404170000.log - jobid: 109 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411270000.log - jobid: 333 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403260000/config.yaml, output/data/runs/b30a-0209f019/202403260000/resources, output/data/runs/b30a-0209f019/202403260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403260000.log - jobid: 620 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405210000/config.yaml, output/data/runs/b30a-0209f019/202405210000/resources, output/data/runs/b30a-0209f019/202405210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405210000.log - jobid: 788 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412030000/config.yaml, output/data/runs/b30a-0209f019/202412030000/resources, output/data/runs/b30a-0209f019/202412030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412030000.log - jobid: 1376 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408080000.log - jobid: 222 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406180000/config.yaml, output/data/runs/b30a-0209f019/202406180000/resources, output/data/runs/b30a-0209f019/202406180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406180000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406180000.log - jobid: 872 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402220000.log - jobid: 54 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404180000.log - jobid: 110 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202401130000/config.yaml, output/data/runs/b30a-0209f019/202401130000/resources, output/data/runs/b30a-0209f019/202401130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401130000.log - jobid: 401 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202401130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411280000.log - jobid: 334 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408090000.log - jobid: 223 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407190000/config.yaml, output/data/runs/b30a-0209f019/202407190000/resources, output/data/runs/b30a-0209f019/202407190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407190000.log - jobid: 965 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402230000.log - jobid: 55 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404190000.log - jobid: 111 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411290000.log - jobid: 335 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410220000/config.yaml, output/data/runs/b30a-0209f019/202410220000/resources, output/data/runs/b30a-0209f019/202410220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410220000.log - jobid: 1250 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405220000/config.yaml, output/data/runs/b30a-0209f019/202405220000/resources, output/data/runs/b30a-0209f019/202405220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405220000.log - jobid: 791 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403270000/config.yaml, output/data/runs/b30a-0209f019/202403270000/resources, output/data/runs/b30a-0209f019/202403270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403270000.log - jobid: 623 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412040000/config.yaml, output/data/runs/b30a-0209f019/202412040000/resources, output/data/runs/b30a-0209f019/202412040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412040000.log - jobid: 1379 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408100000.log - jobid: 224 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406190000/config.yaml, output/data/runs/b30a-0209f019/202406190000/resources, output/data/runs/b30a-0209f019/202406190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406190000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406190000.log - jobid: 875 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411120000/config.yaml, output/data/runs/b30a-0209f019/202411120000/resources, output/data/runs/b30a-0209f019/202411120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411120000.log - jobid: 1313 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402240000.log - jobid: 56 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411300000.log - jobid: 336 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202411300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404200000.log - jobid: 112 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408110000.log - jobid: 225 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402250000.log - jobid: 57 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404210000.log - jobid: 113 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412010000.log - jobid: 337 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403280000/config.yaml, output/data/runs/b30a-0209f019/202403280000/resources, output/data/runs/b30a-0209f019/202403280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403280000.log - jobid: 626 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405230000/config.yaml, output/data/runs/b30a-0209f019/202405230000/resources, output/data/runs/b30a-0209f019/202405230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405230000.log - jobid: 794 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412050000/config.yaml, output/data/runs/b30a-0209f019/202412050000/resources, output/data/runs/b30a-0209f019/202412050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412050000.log - jobid: 1382 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408120000.log - jobid: 226 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406200000/config.yaml, output/data/runs/b30a-0209f019/202406200000/resources, output/data/runs/b30a-0209f019/202406200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406200000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406200000.log - jobid: 878 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402260000.log - jobid: 58 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404220000.log - jobid: 114 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412020000.log - jobid: 338 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410110000/config.yaml, output/data/runs/b30a-0209f019/202410110000/resources, output/data/runs/b30a-0209f019/202410110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410110000.log - jobid: 1217 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408130000.log - jobid: 227 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402270000.log - jobid: 59 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404230000.log - jobid: 115 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412030000.log - jobid: 339 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410290000.log - jobid: 304 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403290000/config.yaml, output/data/runs/b30a-0209f019/202403290000/resources, output/data/runs/b30a-0209f019/202403290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403290000.log - jobid: 629 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405240000/config.yaml, output/data/runs/b30a-0209f019/202405240000/resources, output/data/runs/b30a-0209f019/202405240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405240000.log - jobid: 797 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412060000/config.yaml, output/data/runs/b30a-0209f019/202412060000/resources, output/data/runs/b30a-0209f019/202412060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412060000.log - jobid: 1385 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408140000.log - jobid: 228 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406210000/config.yaml, output/data/runs/b30a-0209f019/202406210000/resources, output/data/runs/b30a-0209f019/202406210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406210000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406210000.log - jobid: 881 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402280000.log - jobid: 60 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404240000.log - jobid: 116 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412040000.log - jobid: 340 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408150000.log - jobid: 229 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402290000.log - jobid: 61 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202402290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404250000.log - jobid: 117 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412050000.log - jobid: 341 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403300000/config.yaml, output/data/runs/b30a-0209f019/202403300000/resources, output/data/runs/b30a-0209f019/202403300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403300000.log - jobid: 632 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405250000/config.yaml, output/data/runs/b30a-0209f019/202405250000/resources, output/data/runs/b30a-0209f019/202405250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405250000.log - jobid: 800 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412070000/config.yaml, output/data/runs/b30a-0209f019/202412070000/resources, output/data/runs/b30a-0209f019/202412070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412070000.log - jobid: 1388 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408160000.log - jobid: 230 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406220000/config.yaml, output/data/runs/b30a-0209f019/202406220000/resources, output/data/runs/b30a-0209f019/202406220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406220000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406220000.log - jobid: 884 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403010000.log - jobid: 62 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404260000.log - jobid: 118 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412060000.log - jobid: 342 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408170000.log - jobid: 231 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403020000.log - jobid: 63 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404270000.log - jobid: 119 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412070000.log - jobid: 343 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202403310000/config.yaml, output/data/runs/b30a-0209f019/202403310000/resources, output/data/runs/b30a-0209f019/202403310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403310000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403310000.log - jobid: 635 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202403310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405260000/config.yaml, output/data/runs/b30a-0209f019/202405260000/resources, output/data/runs/b30a-0209f019/202405260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405260000.log - jobid: 803 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412080000/config.yaml, output/data/runs/b30a-0209f019/202412080000/resources, output/data/runs/b30a-0209f019/202412080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412080000.log - jobid: 1391 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408180000.log - jobid: 232 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406230000/config.yaml, output/data/runs/b30a-0209f019/202406230000/resources, output/data/runs/b30a-0209f019/202406230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406230000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406230000.log - jobid: 887 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403030000.log - jobid: 64 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404280000.log - jobid: 120 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412080000.log - jobid: 344 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408190000.log - jobid: 233 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403040000.log - jobid: 65 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404290000.log - jobid: 121 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412090000.log - jobid: 345 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404010000/config.yaml, output/data/runs/b30a-0209f019/202404010000/resources, output/data/runs/b30a-0209f019/202404010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404010000.log - jobid: 638 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405270000/config.yaml, output/data/runs/b30a-0209f019/202405270000/resources, output/data/runs/b30a-0209f019/202405270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405270000.log - jobid: 806 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412090000/config.yaml, output/data/runs/b30a-0209f019/202412090000/resources, output/data/runs/b30a-0209f019/202412090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412090000.log - jobid: 1394 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408200000.log - jobid: 234 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406240000/config.yaml, output/data/runs/b30a-0209f019/202406240000/resources, output/data/runs/b30a-0209f019/202406240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406240000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406240000.log - jobid: 890 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411130000/config.yaml, output/data/runs/b30a-0209f019/202411130000/resources, output/data/runs/b30a-0209f019/202411130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411130000.log - jobid: 1316 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403050000.log - jobid: 66 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404300000.log - jobid: 122 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202404300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412100000.log - jobid: 346 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408210000.log - jobid: 235 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403060000.log - jobid: 67 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405010000.log - jobid: 123 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412110000.log - jobid: 347 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404020000/config.yaml, output/data/runs/b30a-0209f019/202404020000/resources, output/data/runs/b30a-0209f019/202404020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404020000.log - jobid: 641 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405280000/config.yaml, output/data/runs/b30a-0209f019/202405280000/resources, output/data/runs/b30a-0209f019/202405280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405280000.log - jobid: 809 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412100000/config.yaml, output/data/runs/b30a-0209f019/202412100000/resources, output/data/runs/b30a-0209f019/202412100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412100000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412100000.log - jobid: 1397 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408220000.log - jobid: 236 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406250000/config.yaml, output/data/runs/b30a-0209f019/202406250000/resources, output/data/runs/b30a-0209f019/202406250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406250000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406250000.log - jobid: 893 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403070000.log - jobid: 68 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405020000.log - jobid: 124 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412120000.log - jobid: 348 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202410120000/config.yaml, output/data/runs/b30a-0209f019/202410120000/resources, output/data/runs/b30a-0209f019/202410120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410120000.log - jobid: 1220 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202410120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408230000.log - jobid: 237 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403080000.log - jobid: 69 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405030000.log - jobid: 125 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412130000.log - jobid: 349 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410310000.log - jobid: 306 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202410310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404030000/config.yaml, output/data/runs/b30a-0209f019/202404030000/resources, output/data/runs/b30a-0209f019/202404030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404030000.log - jobid: 644 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405290000/config.yaml, output/data/runs/b30a-0209f019/202405290000/resources, output/data/runs/b30a-0209f019/202405290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405290000.log - jobid: 812 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412110000/config.yaml, output/data/runs/b30a-0209f019/202412110000/resources, output/data/runs/b30a-0209f019/202412110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412110000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412110000.log - jobid: 1400 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408240000.log - jobid: 238 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406260000/config.yaml, output/data/runs/b30a-0209f019/202406260000/resources, output/data/runs/b30a-0209f019/202406260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406260000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406260000.log - jobid: 896 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403090000.log - jobid: 70 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405040000.log - jobid: 126 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412140000.log - jobid: 350 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408250000.log - jobid: 239 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403100000.log - jobid: 71 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405050000.log - jobid: 127 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412150000.log - jobid: 351 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404040000/config.yaml, output/data/runs/b30a-0209f019/202404040000/resources, output/data/runs/b30a-0209f019/202404040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404040000.log - jobid: 647 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405300000/config.yaml, output/data/runs/b30a-0209f019/202405300000/resources, output/data/runs/b30a-0209f019/202405300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405300000.log - jobid: 815 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412120000/config.yaml, output/data/runs/b30a-0209f019/202412120000/resources, output/data/runs/b30a-0209f019/202412120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412120000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412120000.log - jobid: 1403 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408260000.log - jobid: 240 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406270000/config.yaml, output/data/runs/b30a-0209f019/202406270000/resources, output/data/runs/b30a-0209f019/202406270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406270000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406270000.log - jobid: 899 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403110000.log - jobid: 72 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405060000.log - jobid: 128 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412160000.log - jobid: 352 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408270000.log - jobid: 241 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403120000.log - jobid: 73 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405070000.log - jobid: 129 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412170000.log - jobid: 353 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404050000/config.yaml, output/data/runs/b30a-0209f019/202404050000/resources, output/data/runs/b30a-0209f019/202404050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404050000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404050000.log - jobid: 650 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202405310000/config.yaml, output/data/runs/b30a-0209f019/202405310000/resources, output/data/runs/b30a-0209f019/202405310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405310000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405310000.log - jobid: 818 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202405310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412130000/config.yaml, output/data/runs/b30a-0209f019/202412130000/resources, output/data/runs/b30a-0209f019/202412130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412130000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412130000.log - jobid: 1406 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408280000.log - jobid: 242 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406280000/config.yaml, output/data/runs/b30a-0209f019/202406280000/resources, output/data/runs/b30a-0209f019/202406280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406280000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406280000.log - jobid: 902 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403130000.log - jobid: 74 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405080000.log - jobid: 130 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412180000.log - jobid: 354 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408290000.log - jobid: 243 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403140000.log - jobid: 75 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405090000.log - jobid: 131 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412190000.log - jobid: 355 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404060000/config.yaml, output/data/runs/b30a-0209f019/202404060000/resources, output/data/runs/b30a-0209f019/202404060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404060000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404060000.log - jobid: 653 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406010000/config.yaml, output/data/runs/b30a-0209f019/202406010000/resources, output/data/runs/b30a-0209f019/202406010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406010000.log - jobid: 821 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412140000/config.yaml, output/data/runs/b30a-0209f019/202412140000/resources, output/data/runs/b30a-0209f019/202412140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412140000.log - jobid: 1409 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408300000.log - jobid: 244 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406290000/config.yaml, output/data/runs/b30a-0209f019/202406290000/resources, output/data/runs/b30a-0209f019/202406290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406290000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406290000.log - jobid: 905 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411140000/config.yaml, output/data/runs/b30a-0209f019/202411140000/resources, output/data/runs/b30a-0209f019/202411140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411140000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411140000.log - jobid: 1319 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403150000.log - jobid: 76 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405100000.log - jobid: 132 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412200000.log - jobid: 356 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408310000.log - jobid: 245 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202408310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403160000.log - jobid: 77 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405110000.log - jobid: 133 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412210000.log - jobid: 357 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404070000/config.yaml, output/data/runs/b30a-0209f019/202404070000/resources, output/data/runs/b30a-0209f019/202404070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404070000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404070000.log - jobid: 656 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406020000/config.yaml, output/data/runs/b30a-0209f019/202406020000/resources, output/data/runs/b30a-0209f019/202406020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406020000.log - jobid: 824 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412150000/config.yaml, output/data/runs/b30a-0209f019/202412150000/resources, output/data/runs/b30a-0209f019/202412150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412150000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412150000.log - jobid: 1412 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409010000.log - jobid: 246 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406300000/config.yaml, output/data/runs/b30a-0209f019/202406300000/resources, output/data/runs/b30a-0209f019/202406300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406300000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406300000.log - jobid: 908 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403170000.log - jobid: 78 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405120000.log - jobid: 134 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412220000.log - jobid: 358 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409020000.log - jobid: 247 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403180000.log - jobid: 79 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405130000.log - jobid: 135 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412230000.log - jobid: 359 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202411030000/config.yaml, output/data/runs/b30a-0209f019/202411030000/resources, output/data/runs/b30a-0209f019/202411030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411030000.log - jobid: 1286 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202411030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404080000/config.yaml, output/data/runs/b30a-0209f019/202404080000/resources, output/data/runs/b30a-0209f019/202404080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404080000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404080000.log - jobid: 659 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406030000/config.yaml, output/data/runs/b30a-0209f019/202406030000/resources, output/data/runs/b30a-0209f019/202406030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406030000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406030000.log - jobid: 827 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412160000/config.yaml, output/data/runs/b30a-0209f019/202412160000/resources, output/data/runs/b30a-0209f019/202412160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412160000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412160000.log - jobid: 1415 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409030000.log - jobid: 248 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407010000/config.yaml, output/data/runs/b30a-0209f019/202407010000/resources, output/data/runs/b30a-0209f019/202407010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407010000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407010000.log - jobid: 911 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403190000.log - jobid: 80 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405140000.log - jobid: 136 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412240000.log - jobid: 360 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409040000.log - jobid: 249 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403200000.log - jobid: 81 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405150000.log - jobid: 137 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412250000.log - jobid: 361 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202412250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202404090000/config.yaml, output/data/runs/b30a-0209f019/202404090000/resources, output/data/runs/b30a-0209f019/202404090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404090000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404090000.log - jobid: 662 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202404090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202406040000/config.yaml, output/data/runs/b30a-0209f019/202406040000/resources, output/data/runs/b30a-0209f019/202406040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406040000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406040000.log - jobid: 830 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202406040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202412170000/config.yaml, output/data/runs/b30a-0209f019/202412170000/resources, output/data/runs/b30a-0209f019/202412170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412170000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412170000.log - jobid: 1418 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202412170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409050000.log - jobid: 250 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule prepare_inference_forecaster: - input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml - output: output/data/runs/b30a-0209f019/202407020000/config.yaml, output/data/runs/b30a-0209f019/202407020000/resources, output/data/runs/b30a-0209f019/202407020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407020000.ok - log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407020000.log - jobid: 914 - reason: Code has changed since last execution - wildcards: run_id=b30a-0209f019, init_time=202407020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403210000.log - jobid: 82 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405160000.log - jobid: 138 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409060000.log - jobid: 251 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202409060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403220000.log - jobid: 83 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202403220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_baseline: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc - log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405170000.log - jobid: 139 - reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py - wildcards: baseline_id=ICON-CH1-EPS, init_time=202405170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -Would remove temporary output output/data/runs/b30a-0209f019/.venv - -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401190000.ok - log: output/logs/execute_inference/b30a-0209f019-202401190000.log - jobid: 418 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408020000.ok - log: output/logs/execute_inference/b30a-0209f019-202408020000.log - jobid: 1006 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408020000.ok - wildcards: run_id=b30a-0209f019, init_time=202408020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407050000.ok - log: output/logs/execute_inference/b30a-0209f019-202407050000.log - jobid: 922 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407050000.ok - wildcards: run_id=b30a-0209f019, init_time=202407050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404120000.ok - log: output/logs/execute_inference/b30a-0209f019-202404120000.log - jobid: 670 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404120000.ok - wildcards: run_id=b30a-0209f019, init_time=202404120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402160000.ok - log: output/logs/execute_inference/b30a-0209f019-202402160000.log - jobid: 502 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402160000.ok - wildcards: run_id=b30a-0209f019, init_time=202402160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408300000.ok - log: output/logs/execute_inference/b30a-0209f019-202408300000.log - jobid: 1090 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202408300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409270000.ok - log: output/logs/execute_inference/b30a-0209f019-202409270000.log - jobid: 1174 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409270000.ok - wildcards: run_id=b30a-0209f019, init_time=202409270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410250000.ok - log: output/logs/execute_inference/b30a-0209f019-202410250000.log - jobid: 1258 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408310000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408310000.ok - log: output/logs/execute_inference/b30a-0209f019-202408310000.log - jobid: 1093 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408310000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202408310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402170000.ok - log: output/logs/execute_inference/b30a-0209f019-202402170000.log - jobid: 505 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202402170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401200000.ok - log: output/logs/execute_inference/b30a-0209f019-202401200000.log - jobid: 421 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401200000.ok - wildcards: run_id=b30a-0209f019, init_time=202401200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404130000.ok - log: output/logs/execute_inference/b30a-0209f019-202404130000.log - jobid: 673 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404130000.ok - wildcards: run_id=b30a-0209f019, init_time=202404130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407060000.ok - log: output/logs/execute_inference/b30a-0209f019-202407060000.log - jobid: 925 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407060000.ok - wildcards: run_id=b30a-0209f019, init_time=202407060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408030000.ok - log: output/logs/execute_inference/b30a-0209f019-202408030000.log - jobid: 1009 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408030000.ok - wildcards: run_id=b30a-0209f019, init_time=202408030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409280000.ok - log: output/logs/execute_inference/b30a-0209f019-202409280000.log - jobid: 1177 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410260000.ok - log: output/logs/execute_inference/b30a-0209f019-202410260000.log - jobid: 1261 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410260000.ok - wildcards: run_id=b30a-0209f019, init_time=202410260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404140000.ok - log: output/logs/execute_inference/b30a-0209f019-202404140000.log - jobid: 676 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404140000.ok - wildcards: run_id=b30a-0209f019, init_time=202404140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401210000.ok - log: output/logs/execute_inference/b30a-0209f019-202401210000.log - jobid: 424 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401210000.ok - wildcards: run_id=b30a-0209f019, init_time=202401210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402180000.ok - log: output/logs/execute_inference/b30a-0209f019-202402180000.log - jobid: 508 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402180000.ok - wildcards: run_id=b30a-0209f019, init_time=202402180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407070000.ok - log: output/logs/execute_inference/b30a-0209f019-202407070000.log - jobid: 928 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408040000.ok - log: output/logs/execute_inference/b30a-0209f019-202408040000.log - jobid: 1012 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408040000.ok - wildcards: run_id=b30a-0209f019, init_time=202408040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409010000.ok - log: output/logs/execute_inference/b30a-0209f019-202409010000.log - jobid: 1096 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409010000.ok - wildcards: run_id=b30a-0209f019, init_time=202409010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409290000.ok - log: output/logs/execute_inference/b30a-0209f019-202409290000.log - jobid: 1180 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410270000.ok - log: output/logs/execute_inference/b30a-0209f019-202410270000.log - jobid: 1264 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410270000.ok - wildcards: run_id=b30a-0209f019, init_time=202410270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401220000.ok - log: output/logs/execute_inference/b30a-0209f019-202401220000.log - jobid: 427 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401220000.ok - wildcards: run_id=b30a-0209f019, init_time=202401220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402190000.ok - log: output/logs/execute_inference/b30a-0209f019-202402190000.log - jobid: 511 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402190000.ok - wildcards: run_id=b30a-0209f019, init_time=202402190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404150000.ok - log: output/logs/execute_inference/b30a-0209f019-202404150000.log - jobid: 679 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202404150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407080000.ok - log: output/logs/execute_inference/b30a-0209f019-202407080000.log - jobid: 931 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407080000.ok - wildcards: run_id=b30a-0209f019, init_time=202407080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408050000.ok - log: output/logs/execute_inference/b30a-0209f019-202408050000.log - jobid: 1015 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408050000.ok - wildcards: run_id=b30a-0209f019, init_time=202408050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409020000.ok - log: output/logs/execute_inference/b30a-0209f019-202409020000.log - jobid: 1099 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409020000.ok - wildcards: run_id=b30a-0209f019, init_time=202409020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409300000.ok - log: output/logs/execute_inference/b30a-0209f019-202409300000.log - jobid: 1183 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409300000.ok - wildcards: run_id=b30a-0209f019, init_time=202409300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410280000.ok - log: output/logs/execute_inference/b30a-0209f019-202410280000.log - jobid: 1267 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410280000.ok - wildcards: run_id=b30a-0209f019, init_time=202410280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401230000.ok - log: output/logs/execute_inference/b30a-0209f019-202401230000.log - jobid: 430 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402200000.ok - log: output/logs/execute_inference/b30a-0209f019-202402200000.log - jobid: 514 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402200000.ok - wildcards: run_id=b30a-0209f019, init_time=202402200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404160000.ok - log: output/logs/execute_inference/b30a-0209f019-202404160000.log - jobid: 682 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404160000.ok - wildcards: run_id=b30a-0209f019, init_time=202404160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407090000.ok - log: output/logs/execute_inference/b30a-0209f019-202407090000.log - jobid: 934 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407090000.ok - wildcards: run_id=b30a-0209f019, init_time=202407090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408060000.ok - log: output/logs/execute_inference/b30a-0209f019-202408060000.log - jobid: 1018 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408060000.ok - wildcards: run_id=b30a-0209f019, init_time=202408060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409030000.ok - log: output/logs/execute_inference/b30a-0209f019-202409030000.log - jobid: 1102 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410010000.ok - log: output/logs/execute_inference/b30a-0209f019-202410010000.log - jobid: 1186 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410010000.ok - wildcards: run_id=b30a-0209f019, init_time=202410010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410290000.ok - log: output/logs/execute_inference/b30a-0209f019-202410290000.log - jobid: 1270 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401240000.ok - log: output/logs/execute_inference/b30a-0209f019-202401240000.log - jobid: 433 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402210000.ok - log: output/logs/execute_inference/b30a-0209f019-202402210000.log - jobid: 517 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202402210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404170000.ok - log: output/logs/execute_inference/b30a-0209f019-202404170000.log - jobid: 685 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404170000.ok - wildcards: run_id=b30a-0209f019, init_time=202404170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407100000.ok - log: output/logs/execute_inference/b30a-0209f019-202407100000.log - jobid: 937 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407100000.ok - wildcards: run_id=b30a-0209f019, init_time=202407100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408070000.ok - log: output/logs/execute_inference/b30a-0209f019-202408070000.log - jobid: 1021 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202408070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409040000.ok - log: output/logs/execute_inference/b30a-0209f019-202409040000.log - jobid: 1105 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410020000.ok - log: output/logs/execute_inference/b30a-0209f019-202410020000.log - jobid: 1189 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410020000.ok - wildcards: run_id=b30a-0209f019, init_time=202410020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410300000.ok - log: output/logs/execute_inference/b30a-0209f019-202410300000.log - jobid: 1273 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401250000.ok - log: output/logs/execute_inference/b30a-0209f019-202401250000.log - jobid: 436 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402220000.ok - log: output/logs/execute_inference/b30a-0209f019-202402220000.log - jobid: 520 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402220000.ok - wildcards: run_id=b30a-0209f019, init_time=202402220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404180000.ok - log: output/logs/execute_inference/b30a-0209f019-202404180000.log - jobid: 688 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202404180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407110000.ok - log: output/logs/execute_inference/b30a-0209f019-202407110000.log - jobid: 940 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407110000.ok - wildcards: run_id=b30a-0209f019, init_time=202407110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408080000.ok - log: output/logs/execute_inference/b30a-0209f019-202408080000.log - jobid: 1024 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408080000.ok - wildcards: run_id=b30a-0209f019, init_time=202408080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409050000.ok - log: output/logs/execute_inference/b30a-0209f019-202409050000.log - jobid: 1108 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409050000.ok - wildcards: run_id=b30a-0209f019, init_time=202409050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410030000.ok - log: output/logs/execute_inference/b30a-0209f019-202410030000.log - jobid: 1192 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410310000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410310000.ok - log: output/logs/execute_inference/b30a-0209f019-202410310000.log - jobid: 1276 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410310000.ok - wildcards: run_id=b30a-0209f019, init_time=202410310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401260000.ok - log: output/logs/execute_inference/b30a-0209f019-202401260000.log - jobid: 439 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401260000.ok - wildcards: run_id=b30a-0209f019, init_time=202401260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402230000.ok - log: output/logs/execute_inference/b30a-0209f019-202402230000.log - jobid: 523 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402230000.ok - wildcards: run_id=b30a-0209f019, init_time=202402230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404190000.ok - log: output/logs/execute_inference/b30a-0209f019-202404190000.log - jobid: 691 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202404190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407120000.ok - log: output/logs/execute_inference/b30a-0209f019-202407120000.log - jobid: 943 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410040000.ok - log: output/logs/execute_inference/b30a-0209f019-202410040000.log - jobid: 1195 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410040000.ok - wildcards: run_id=b30a-0209f019, init_time=202410040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411010000.ok - log: output/logs/execute_inference/b30a-0209f019-202411010000.log - jobid: 1279 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411010000.ok - wildcards: run_id=b30a-0209f019, init_time=202411010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408090000.ok - log: output/logs/execute_inference/b30a-0209f019-202408090000.log - jobid: 1027 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408090000.ok - wildcards: run_id=b30a-0209f019, init_time=202408090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409060000.ok - log: output/logs/execute_inference/b30a-0209f019-202409060000.log - jobid: 1111 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409060000.ok - wildcards: run_id=b30a-0209f019, init_time=202409060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404200000.ok - log: output/logs/execute_inference/b30a-0209f019-202404200000.log - jobid: 694 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404200000.ok - wildcards: run_id=b30a-0209f019, init_time=202404200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407130000.ok - log: output/logs/execute_inference/b30a-0209f019-202407130000.log - jobid: 946 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408100000.ok - log: output/logs/execute_inference/b30a-0209f019-202408100000.log - jobid: 1030 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408100000.ok - wildcards: run_id=b30a-0209f019, init_time=202408100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409070000.ok - log: output/logs/execute_inference/b30a-0209f019-202409070000.log - jobid: 1114 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401270000.ok - log: output/logs/execute_inference/b30a-0209f019-202401270000.log - jobid: 442 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401270000.ok - wildcards: run_id=b30a-0209f019, init_time=202401270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402240000.ok - log: output/logs/execute_inference/b30a-0209f019-202402240000.log - jobid: 526 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402240000.ok - wildcards: run_id=b30a-0209f019, init_time=202402240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410050000.ok - log: output/logs/execute_inference/b30a-0209f019-202410050000.log - jobid: 1198 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410050000.ok - wildcards: run_id=b30a-0209f019, init_time=202410050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411020000.ok - log: output/logs/execute_inference/b30a-0209f019-202411020000.log - jobid: 1282 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411020000.ok - wildcards: run_id=b30a-0209f019, init_time=202411020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402250000.ok - log: output/logs/execute_inference/b30a-0209f019-202402250000.log - jobid: 529 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202402250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401280000.ok - log: output/logs/execute_inference/b30a-0209f019-202401280000.log - jobid: 445 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401280000.ok - wildcards: run_id=b30a-0209f019, init_time=202401280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404210000.ok - log: output/logs/execute_inference/b30a-0209f019-202404210000.log - jobid: 697 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404210000.ok - wildcards: run_id=b30a-0209f019, init_time=202404210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408110000.ok - log: output/logs/execute_inference/b30a-0209f019-202408110000.log - jobid: 1033 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408110000.ok - wildcards: run_id=b30a-0209f019, init_time=202408110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409080000.ok - log: output/logs/execute_inference/b30a-0209f019-202409080000.log - jobid: 1117 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407140000.ok - log: output/logs/execute_inference/b30a-0209f019-202407140000.log - jobid: 949 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407140000.ok - wildcards: run_id=b30a-0209f019, init_time=202407140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410060000.ok - log: output/logs/execute_inference/b30a-0209f019-202410060000.log - jobid: 1201 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410060000.ok - wildcards: run_id=b30a-0209f019, init_time=202410060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411030000.ok - log: output/logs/execute_inference/b30a-0209f019-202411030000.log - jobid: 1285 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408120000.ok - log: output/logs/execute_inference/b30a-0209f019-202408120000.log - jobid: 1036 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408120000.ok - wildcards: run_id=b30a-0209f019, init_time=202408120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401290000.ok - log: output/logs/execute_inference/b30a-0209f019-202401290000.log - jobid: 448 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409090000.ok - log: output/logs/execute_inference/b30a-0209f019-202409090000.log - jobid: 1120 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409090000.ok - wildcards: run_id=b30a-0209f019, init_time=202409090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402260000.ok - log: output/logs/execute_inference/b30a-0209f019-202402260000.log - jobid: 532 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202402260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404220000.ok - log: output/logs/execute_inference/b30a-0209f019-202404220000.log - jobid: 700 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404220000.ok - wildcards: run_id=b30a-0209f019, init_time=202404220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407150000.ok - log: output/logs/execute_inference/b30a-0209f019-202407150000.log - jobid: 952 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407150000.ok - wildcards: run_id=b30a-0209f019, init_time=202407150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410070000.ok - log: output/logs/execute_inference/b30a-0209f019-202410070000.log - jobid: 1204 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410070000.ok - wildcards: run_id=b30a-0209f019, init_time=202410070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411040000.ok - log: output/logs/execute_inference/b30a-0209f019-202411040000.log - jobid: 1288 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411040000.ok - wildcards: run_id=b30a-0209f019, init_time=202411040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408130000.ok - log: output/logs/execute_inference/b30a-0209f019-202408130000.log - jobid: 1039 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202408130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401300000.ok - log: output/logs/execute_inference/b30a-0209f019-202401300000.log - jobid: 451 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409100000.ok - log: output/logs/execute_inference/b30a-0209f019-202409100000.log - jobid: 1123 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409100000.ok - wildcards: run_id=b30a-0209f019, init_time=202409100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402270000.ok - log: output/logs/execute_inference/b30a-0209f019-202402270000.log - jobid: 535 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402270000.ok - wildcards: run_id=b30a-0209f019, init_time=202402270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404230000.ok - log: output/logs/execute_inference/b30a-0209f019-202404230000.log - jobid: 703 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404230000.ok - wildcards: run_id=b30a-0209f019, init_time=202404230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407160000.ok - log: output/logs/execute_inference/b30a-0209f019-202407160000.log - jobid: 955 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407160000.ok - wildcards: run_id=b30a-0209f019, init_time=202407160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410080000.ok - log: output/logs/execute_inference/b30a-0209f019-202410080000.log - jobid: 1207 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411050000.ok - log: output/logs/execute_inference/b30a-0209f019-202411050000.log - jobid: 1291 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411050000.ok - wildcards: run_id=b30a-0209f019, init_time=202411050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402280000.ok - log: output/logs/execute_inference/b30a-0209f019-202402280000.log - jobid: 538 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402280000.ok - wildcards: run_id=b30a-0209f019, init_time=202402280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404240000.ok - log: output/logs/execute_inference/b30a-0209f019-202404240000.log - jobid: 706 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202404240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407170000.ok - log: output/logs/execute_inference/b30a-0209f019-202407170000.log - jobid: 958 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401310000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401310000.ok - log: output/logs/execute_inference/b30a-0209f019-202401310000.log - jobid: 454 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401310000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409110000.ok - log: output/logs/execute_inference/b30a-0209f019-202409110000.log - jobid: 1126 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409110000.ok - wildcards: run_id=b30a-0209f019, init_time=202409110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411060000.ok - log: output/logs/execute_inference/b30a-0209f019-202411060000.log - jobid: 1294 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411060000.ok - wildcards: run_id=b30a-0209f019, init_time=202411060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408140000.ok - log: output/logs/execute_inference/b30a-0209f019-202408140000.log - jobid: 1042 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408140000.ok - wildcards: run_id=b30a-0209f019, init_time=202408140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410090000.ok - log: output/logs/execute_inference/b30a-0209f019-202410090000.log - jobid: 1210 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410090000.ok - wildcards: run_id=b30a-0209f019, init_time=202410090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401030000.ok - log: output/logs/execute_inference/b30a-0209f019-202401030000.log - jobid: 370 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402010000.ok - log: output/logs/execute_inference/b30a-0209f019-202402010000.log - jobid: 457 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402010000.ok - wildcards: run_id=b30a-0209f019, init_time=202402010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408150000.ok - log: output/logs/execute_inference/b30a-0209f019-202408150000.log - jobid: 1045 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408150000.ok - wildcards: run_id=b30a-0209f019, init_time=202408150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409120000.ok - log: output/logs/execute_inference/b30a-0209f019-202409120000.log - jobid: 1129 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409120000.ok - wildcards: run_id=b30a-0209f019, init_time=202409120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402290000.ok - log: output/logs/execute_inference/b30a-0209f019-202402290000.log - jobid: 541 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402290000.ok - wildcards: run_id=b30a-0209f019, init_time=202402290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401040000.ok - log: output/logs/execute_inference/b30a-0209f019-202401040000.log - jobid: 373 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401040000.ok - wildcards: run_id=b30a-0209f019, init_time=202401040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404250000.ok - log: output/logs/execute_inference/b30a-0209f019-202404250000.log - jobid: 709 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202404250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407180000.ok - log: output/logs/execute_inference/b30a-0209f019-202407180000.log - jobid: 961 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407180000.ok - wildcards: run_id=b30a-0209f019, init_time=202407180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411070000.ok - log: output/logs/execute_inference/b30a-0209f019-202411070000.log - jobid: 1297 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411070000.ok - wildcards: run_id=b30a-0209f019, init_time=202411070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410100000.ok - log: output/logs/execute_inference/b30a-0209f019-202410100000.log - jobid: 1213 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409130000.ok - log: output/logs/execute_inference/b30a-0209f019-202409130000.log - jobid: 1132 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409130000.ok - wildcards: run_id=b30a-0209f019, init_time=202409130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402020000.ok - log: output/logs/execute_inference/b30a-0209f019-202402020000.log - jobid: 460 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202402020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408160000.ok - log: output/logs/execute_inference/b30a-0209f019-202408160000.log - jobid: 1048 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408160000.ok - wildcards: run_id=b30a-0209f019, init_time=202408160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403010000.ok - log: output/logs/execute_inference/b30a-0209f019-202403010000.log - jobid: 544 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403010000.ok - wildcards: run_id=b30a-0209f019, init_time=202403010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401050000.ok - log: output/logs/execute_inference/b30a-0209f019-202401050000.log - jobid: 376 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401050000.ok - wildcards: run_id=b30a-0209f019, init_time=202401050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404260000.ok - log: output/logs/execute_inference/b30a-0209f019-202404260000.log - jobid: 712 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404260000.ok - wildcards: run_id=b30a-0209f019, init_time=202404260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411080000.ok - log: output/logs/execute_inference/b30a-0209f019-202411080000.log - jobid: 1300 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407190000.ok - log: output/logs/execute_inference/b30a-0209f019-202407190000.log - jobid: 964 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_aggregation_baseline: - input: workflow/scripts/verif_aggregation.py, output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc - output: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - log: output/logs/verif_metrics_aggregation_baseline/ICON-CH1-EPS.log - jobid: 2 - reason: Input files updated by another job: output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc - wildcards: baseline_id=ICON-CH1-EPS - resources: mem_mb=250000, mem_mib=238419, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=1440, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410110000.ok - log: output/logs/execute_inference/b30a-0209f019-202410110000.log - jobid: 1216 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403020000.ok - log: output/logs/execute_inference/b30a-0209f019-202403020000.log - jobid: 547 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403020000.ok - wildcards: run_id=b30a-0209f019, init_time=202403020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409140000.ok - log: output/logs/execute_inference/b30a-0209f019-202409140000.log - jobid: 1135 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402030000.ok - log: output/logs/execute_inference/b30a-0209f019-202402030000.log - jobid: 463 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402030000.ok - wildcards: run_id=b30a-0209f019, init_time=202402030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408170000.ok - log: output/logs/execute_inference/b30a-0209f019-202408170000.log - jobid: 1051 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408170000.ok - wildcards: run_id=b30a-0209f019, init_time=202408170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404270000.ok - log: output/logs/execute_inference/b30a-0209f019-202404270000.log - jobid: 715 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404270000.ok - wildcards: run_id=b30a-0209f019, init_time=202404270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407200000.ok - log: output/logs/execute_inference/b30a-0209f019-202407200000.log - jobid: 967 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407200000.ok - wildcards: run_id=b30a-0209f019, init_time=202407200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401060000.ok - log: output/logs/execute_inference/b30a-0209f019-202401060000.log - jobid: 379 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401060000.ok - wildcards: run_id=b30a-0209f019, init_time=202401060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411090000.ok - log: output/logs/execute_inference/b30a-0209f019-202411090000.log - jobid: 1303 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411090000.ok - wildcards: run_id=b30a-0209f019, init_time=202411090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410120000.ok - log: output/logs/execute_inference/b30a-0209f019-202410120000.log - jobid: 1219 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410120000.ok - wildcards: run_id=b30a-0209f019, init_time=202410120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409150000.ok - log: output/logs/execute_inference/b30a-0209f019-202409150000.log - jobid: 1138 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402040000.ok - log: output/logs/execute_inference/b30a-0209f019-202402040000.log - jobid: 466 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402040000.ok - wildcards: run_id=b30a-0209f019, init_time=202402040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408180000.ok - log: output/logs/execute_inference/b30a-0209f019-202408180000.log - jobid: 1054 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408180000.ok - wildcards: run_id=b30a-0209f019, init_time=202408180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403030000.ok - log: output/logs/execute_inference/b30a-0209f019-202403030000.log - jobid: 550 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403030000.ok - wildcards: run_id=b30a-0209f019, init_time=202403030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404280000.ok - log: output/logs/execute_inference/b30a-0209f019-202404280000.log - jobid: 718 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404280000.ok - wildcards: run_id=b30a-0209f019, init_time=202404280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407210000.ok - log: output/logs/execute_inference/b30a-0209f019-202407210000.log - jobid: 970 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410130000.ok - log: output/logs/execute_inference/b30a-0209f019-202410130000.log - jobid: 1222 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410130000.ok - wildcards: run_id=b30a-0209f019, init_time=202410130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401140000.ok - log: output/logs/execute_inference/b30a-0209f019-202401140000.log - jobid: 403 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411100000.ok - log: output/logs/execute_inference/b30a-0209f019-202411100000.log - jobid: 1306 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411110000.ok - log: output/logs/execute_inference/b30a-0209f019-202411110000.log - jobid: 1309 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411110000.ok - wildcards: run_id=b30a-0209f019, init_time=202411110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401070000.ok - log: output/logs/execute_inference/b30a-0209f019-202401070000.log - jobid: 382 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401070000.ok - wildcards: run_id=b30a-0209f019, init_time=202401070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409160000.ok - log: output/logs/execute_inference/b30a-0209f019-202409160000.log - jobid: 1141 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409160000.ok - wildcards: run_id=b30a-0209f019, init_time=202409160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402050000.ok - log: output/logs/execute_inference/b30a-0209f019-202402050000.log - jobid: 469 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202402050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408190000.ok - log: output/logs/execute_inference/b30a-0209f019-202408190000.log - jobid: 1057 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408190000.ok - wildcards: run_id=b30a-0209f019, init_time=202408190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403040000.ok - log: output/logs/execute_inference/b30a-0209f019-202403040000.log - jobid: 553 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403040000.ok - wildcards: run_id=b30a-0209f019, init_time=202403040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404290000.ok - log: output/logs/execute_inference/b30a-0209f019-202404290000.log - jobid: 721 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404290000.ok - wildcards: run_id=b30a-0209f019, init_time=202404290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407220000.ok - log: output/logs/execute_inference/b30a-0209f019-202407220000.log - jobid: 973 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407220000.ok - wildcards: run_id=b30a-0209f019, init_time=202407220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410140000.ok - log: output/logs/execute_inference/b30a-0209f019-202410140000.log - jobid: 1225 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401130000.ok - log: output/logs/execute_inference/b30a-0209f019-202401130000.log - jobid: 400 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401130000.ok - wildcards: run_id=b30a-0209f019, init_time=202401130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403050000.ok - log: output/logs/execute_inference/b30a-0209f019-202403050000.log - jobid: 556 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402060000.ok - log: output/logs/execute_inference/b30a-0209f019-202402060000.log - jobid: 472 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402060000.ok - wildcards: run_id=b30a-0209f019, init_time=202402060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409170000.ok - log: output/logs/execute_inference/b30a-0209f019-202409170000.log - jobid: 1144 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409170000.ok - wildcards: run_id=b30a-0209f019, init_time=202409170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408200000.ok - log: output/logs/execute_inference/b30a-0209f019-202408200000.log - jobid: 1060 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202408200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404300000.ok - log: output/logs/execute_inference/b30a-0209f019-202404300000.log - jobid: 724 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404300000.ok - wildcards: run_id=b30a-0209f019, init_time=202404300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407230000.ok - log: output/logs/execute_inference/b30a-0209f019-202407230000.log - jobid: 976 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410150000.ok - log: output/logs/execute_inference/b30a-0209f019-202410150000.log - jobid: 1228 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410150000.ok - wildcards: run_id=b30a-0209f019, init_time=202410150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411120000.ok - log: output/logs/execute_inference/b30a-0209f019-202411120000.log - jobid: 1312 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401120000.ok - log: output/logs/execute_inference/b30a-0209f019-202401120000.log - jobid: 397 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401080000.ok - log: output/logs/execute_inference/b30a-0209f019-202401080000.log - jobid: 385 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408210000.ok - log: output/logs/execute_inference/b30a-0209f019-202408210000.log - jobid: 1063 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408210000.ok - wildcards: run_id=b30a-0209f019, init_time=202408210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402070000.ok - log: output/logs/execute_inference/b30a-0209f019-202402070000.log - jobid: 475 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402070000.ok - wildcards: run_id=b30a-0209f019, init_time=202402070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409180000.ok - log: output/logs/execute_inference/b30a-0209f019-202409180000.log - jobid: 1147 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409180000.ok - wildcards: run_id=b30a-0209f019, init_time=202409180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403060000.ok - log: output/logs/execute_inference/b30a-0209f019-202403060000.log - jobid: 559 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403060000.ok - wildcards: run_id=b30a-0209f019, init_time=202403060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407240000.ok - log: output/logs/execute_inference/b30a-0209f019-202407240000.log - jobid: 979 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405010000.ok - log: output/logs/execute_inference/b30a-0209f019-202405010000.log - jobid: 727 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410160000.ok - log: output/logs/execute_inference/b30a-0209f019-202410160000.log - jobid: 1231 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411130000.ok - log: output/logs/execute_inference/b30a-0209f019-202411130000.log - jobid: 1315 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411130000.ok - wildcards: run_id=b30a-0209f019, init_time=202411130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401090000.ok - log: output/logs/execute_inference/b30a-0209f019-202401090000.log - jobid: 388 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401090000.ok - wildcards: run_id=b30a-0209f019, init_time=202401090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402080000.ok - log: output/logs/execute_inference/b30a-0209f019-202402080000.log - jobid: 478 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402080000.ok - wildcards: run_id=b30a-0209f019, init_time=202402080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409190000.ok - log: output/logs/execute_inference/b30a-0209f019-202409190000.log - jobid: 1150 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409190000.ok - wildcards: run_id=b30a-0209f019, init_time=202409190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408220000.ok - log: output/logs/execute_inference/b30a-0209f019-202408220000.log - jobid: 1066 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408220000.ok - wildcards: run_id=b30a-0209f019, init_time=202408220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403070000.ok - log: output/logs/execute_inference/b30a-0209f019-202403070000.log - jobid: 562 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405020000.ok - log: output/logs/execute_inference/b30a-0209f019-202405020000.log - jobid: 730 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407250000.ok - log: output/logs/execute_inference/b30a-0209f019-202407250000.log - jobid: 982 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407250000.ok - wildcards: run_id=b30a-0209f019, init_time=202407250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410170000.ok - log: output/logs/execute_inference/b30a-0209f019-202410170000.log - jobid: 1234 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410170000.ok - wildcards: run_id=b30a-0209f019, init_time=202410170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411140000.ok - log: output/logs/execute_inference/b30a-0209f019-202411140000.log - jobid: 1318 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411140000.ok - wildcards: run_id=b30a-0209f019, init_time=202411140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401100000.ok - log: output/logs/execute_inference/b30a-0209f019-202401100000.log - jobid: 391 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401100000.ok - wildcards: run_id=b30a-0209f019, init_time=202401100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411150000.ok - log: output/logs/execute_inference/b30a-0209f019-202411150000.log - jobid: 1321 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411150000.ok - wildcards: run_id=b30a-0209f019, init_time=202411150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407260000.ok - log: output/logs/execute_inference/b30a-0209f019-202407260000.log - jobid: 985 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410180000.ok - log: output/logs/execute_inference/b30a-0209f019-202410180000.log - jobid: 1237 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410180000.ok - wildcards: run_id=b30a-0209f019, init_time=202410180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403080000.ok - log: output/logs/execute_inference/b30a-0209f019-202403080000.log - jobid: 565 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408230000.ok - log: output/logs/execute_inference/b30a-0209f019-202408230000.log - jobid: 1069 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202408230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402090000.ok - log: output/logs/execute_inference/b30a-0209f019-202402090000.log - jobid: 481 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402090000.ok - wildcards: run_id=b30a-0209f019, init_time=202402090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409200000.ok - log: output/logs/execute_inference/b30a-0209f019-202409200000.log - jobid: 1153 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405030000.ok - log: output/logs/execute_inference/b30a-0209f019-202405030000.log - jobid: 733 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405030000.ok - wildcards: run_id=b30a-0209f019, init_time=202405030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401110000.ok - log: output/logs/execute_inference/b30a-0209f019-202401110000.log - jobid: 394 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401110000.ok - wildcards: run_id=b30a-0209f019, init_time=202401110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403090000.ok - log: output/logs/execute_inference/b30a-0209f019-202403090000.log - jobid: 568 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403090000.ok - wildcards: run_id=b30a-0209f019, init_time=202403090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408240000.ok - log: output/logs/execute_inference/b30a-0209f019-202408240000.log - jobid: 1072 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408240000.ok - wildcards: run_id=b30a-0209f019, init_time=202408240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402100000.ok - log: output/logs/execute_inference/b30a-0209f019-202402100000.log - jobid: 484 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402100000.ok - wildcards: run_id=b30a-0209f019, init_time=202402100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409210000.ok - log: output/logs/execute_inference/b30a-0209f019-202409210000.log - jobid: 1156 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409210000.ok - wildcards: run_id=b30a-0209f019, init_time=202409210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405040000.ok - log: output/logs/execute_inference/b30a-0209f019-202405040000.log - jobid: 736 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405040000.ok - wildcards: run_id=b30a-0209f019, init_time=202405040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407270000.ok - log: output/logs/execute_inference/b30a-0209f019-202407270000.log - jobid: 988 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202407270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411160000.ok - log: output/logs/execute_inference/b30a-0209f019-202411160000.log - jobid: 1324 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411160000.ok - wildcards: run_id=b30a-0209f019, init_time=202411160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410190000.ok - log: output/logs/execute_inference/b30a-0209f019-202410190000.log - jobid: 1240 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410190000.ok - wildcards: run_id=b30a-0209f019, init_time=202410190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403100000.ok - log: output/logs/execute_inference/b30a-0209f019-202403100000.log - jobid: 571 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408250000.ok - log: output/logs/execute_inference/b30a-0209f019-202408250000.log - jobid: 1075 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202408250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402110000.ok - log: output/logs/execute_inference/b30a-0209f019-202402110000.log - jobid: 487 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202402110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409220000.ok - log: output/logs/execute_inference/b30a-0209f019-202409220000.log - jobid: 1159 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407280000.ok - log: output/logs/execute_inference/b30a-0209f019-202407280000.log - jobid: 991 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407280000.ok - wildcards: run_id=b30a-0209f019, init_time=202407280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405050000.ok - log: output/logs/execute_inference/b30a-0209f019-202405050000.log - jobid: 739 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405050000.ok - wildcards: run_id=b30a-0209f019, init_time=202405050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411170000.ok - log: output/logs/execute_inference/b30a-0209f019-202411170000.log - jobid: 1327 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410200000.ok - log: output/logs/execute_inference/b30a-0209f019-202410200000.log - jobid: 1243 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410200000.ok - wildcards: run_id=b30a-0209f019, init_time=202410200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410210000.ok - log: output/logs/execute_inference/b30a-0209f019-202410210000.log - jobid: 1246 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410210000.ok - wildcards: run_id=b30a-0209f019, init_time=202410210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403110000.ok - log: output/logs/execute_inference/b30a-0209f019-202403110000.log - jobid: 574 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403110000.ok - wildcards: run_id=b30a-0209f019, init_time=202403110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408260000.ok - log: output/logs/execute_inference/b30a-0209f019-202408260000.log - jobid: 1078 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408260000.ok - wildcards: run_id=b30a-0209f019, init_time=202408260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402120000.ok - log: output/logs/execute_inference/b30a-0209f019-202402120000.log - jobid: 490 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402120000.ok - wildcards: run_id=b30a-0209f019, init_time=202402120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409230000.ok - log: output/logs/execute_inference/b30a-0209f019-202409230000.log - jobid: 1162 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405060000.ok - log: output/logs/execute_inference/b30a-0209f019-202405060000.log - jobid: 742 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405060000.ok - wildcards: run_id=b30a-0209f019, init_time=202405060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407290000.ok - log: output/logs/execute_inference/b30a-0209f019-202407290000.log - jobid: 994 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407290000.ok - wildcards: run_id=b30a-0209f019, init_time=202407290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401150000.ok - log: output/logs/execute_inference/b30a-0209f019-202401150000.log - jobid: 406 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411180000.ok - log: output/logs/execute_inference/b30a-0209f019-202411180000.log - jobid: 1330 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410220000.ok - log: output/logs/execute_inference/b30a-0209f019-202410220000.log - jobid: 1249 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411190000.ok - log: output/logs/execute_inference/b30a-0209f019-202411190000.log - jobid: 1333 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409240000.ok - log: output/logs/execute_inference/b30a-0209f019-202409240000.log - jobid: 1165 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409240000.ok - wildcards: run_id=b30a-0209f019, init_time=202409240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408270000.ok - log: output/logs/execute_inference/b30a-0209f019-202408270000.log - jobid: 1081 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408270000.ok - wildcards: run_id=b30a-0209f019, init_time=202408270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402130000.ok - log: output/logs/execute_inference/b30a-0209f019-202402130000.log - jobid: 493 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402130000.ok - wildcards: run_id=b30a-0209f019, init_time=202402130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403120000.ok - log: output/logs/execute_inference/b30a-0209f019-202403120000.log - jobid: 577 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405070000.ok - log: output/logs/execute_inference/b30a-0209f019-202405070000.log - jobid: 745 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407300000.ok - log: output/logs/execute_inference/b30a-0209f019-202407300000.log - jobid: 997 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407300000.ok - wildcards: run_id=b30a-0209f019, init_time=202407300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401160000.ok - log: output/logs/execute_inference/b30a-0209f019-202401160000.log - jobid: 409 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401160000.ok - wildcards: run_id=b30a-0209f019, init_time=202401160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410230000.ok - log: output/logs/execute_inference/b30a-0209f019-202410230000.log - jobid: 1252 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411200000.ok - log: output/logs/execute_inference/b30a-0209f019-202411200000.log - jobid: 1336 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411200000.ok - wildcards: run_id=b30a-0209f019, init_time=202411200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403130000.ok - log: output/logs/execute_inference/b30a-0209f019-202403130000.log - jobid: 580 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403130000.ok - wildcards: run_id=b30a-0209f019, init_time=202403130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405080000.ok - log: output/logs/execute_inference/b30a-0209f019-202405080000.log - jobid: 748 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405080000.ok - wildcards: run_id=b30a-0209f019, init_time=202405080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402140000.ok - log: output/logs/execute_inference/b30a-0209f019-202402140000.log - jobid: 496 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402140000.ok - wildcards: run_id=b30a-0209f019, init_time=202402140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401170000.ok - log: output/logs/execute_inference/b30a-0209f019-202401170000.log - jobid: 412 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401170000.ok - wildcards: run_id=b30a-0209f019, init_time=202401170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407310000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407310000.ok - log: output/logs/execute_inference/b30a-0209f019-202407310000.log - jobid: 1000 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407310000.ok - wildcards: run_id=b30a-0209f019, init_time=202407310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408280000.ok - log: output/logs/execute_inference/b30a-0209f019-202408280000.log - jobid: 1084 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202408280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409250000.ok - log: output/logs/execute_inference/b30a-0209f019-202409250000.log - jobid: 1168 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202410240000.ok - log: output/logs/execute_inference/b30a-0209f019-202410240000.log - jobid: 1255 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202410240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202409260000.ok - log: output/logs/execute_inference/b30a-0209f019-202409260000.log - jobid: 1171 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202409260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405090000.ok - log: output/logs/execute_inference/b30a-0209f019-202405090000.log - jobid: 751 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411210000.ok - log: output/logs/execute_inference/b30a-0209f019-202411210000.log - jobid: 1339 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411210000.ok - wildcards: run_id=b30a-0209f019, init_time=202411210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403140000.ok - log: output/logs/execute_inference/b30a-0209f019-202403140000.log - jobid: 583 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401180000.ok - log: output/logs/execute_inference/b30a-0209f019-202401180000.log - jobid: 415 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401180000.ok - wildcards: run_id=b30a-0209f019, init_time=202401180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202402150000.ok - log: output/logs/execute_inference/b30a-0209f019-202402150000.log - jobid: 499 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402150000.ok - wildcards: run_id=b30a-0209f019, init_time=202402150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408010000.ok - log: output/logs/execute_inference/b30a-0209f019-202408010000.log - jobid: 1003 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202408010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202408290000.ok - log: output/logs/execute_inference/b30a-0209f019-202408290000.log - jobid: 1087 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408290000.ok - wildcards: run_id=b30a-0209f019, init_time=202408290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403150000.ok - log: output/logs/execute_inference/b30a-0209f019-202403150000.log - jobid: 586 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403150000.ok - wildcards: run_id=b30a-0209f019, init_time=202403150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405100000.ok - log: output/logs/execute_inference/b30a-0209f019-202405100000.log - jobid: 754 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405100000.ok - wildcards: run_id=b30a-0209f019, init_time=202405100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412200000.ok - log: output/logs/execute_inference/b30a-0209f019-202412200000.log - jobid: 1426 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412200000.ok - wildcards: run_id=b30a-0209f019, init_time=202412200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411220000.ok - log: output/logs/execute_inference/b30a-0209f019-202411220000.log - jobid: 1342 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411220000.ok - wildcards: run_id=b30a-0209f019, init_time=202411220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406070000.ok - log: output/logs/execute_inference/b30a-0209f019-202406070000.log - jobid: 838 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403160000.ok - log: output/logs/execute_inference/b30a-0209f019-202403160000.log - jobid: 589 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403160000.ok - wildcards: run_id=b30a-0209f019, init_time=202403160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411230000.ok - log: output/logs/execute_inference/b30a-0209f019-202411230000.log - jobid: 1345 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411230000.ok - wildcards: run_id=b30a-0209f019, init_time=202411230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406080000.ok - log: output/logs/execute_inference/b30a-0209f019-202406080000.log - jobid: 841 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406080000.ok - wildcards: run_id=b30a-0209f019, init_time=202406080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405110000.ok - log: output/logs/execute_inference/b30a-0209f019-202405110000.log - jobid: 757 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412210000.ok - log: output/logs/execute_inference/b30a-0209f019-202412210000.log - jobid: 1429 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412210000.ok - wildcards: run_id=b30a-0209f019, init_time=202412210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412220000.ok - log: output/logs/execute_inference/b30a-0209f019-202412220000.log - jobid: 1432 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412220000.ok - wildcards: run_id=b30a-0209f019, init_time=202412220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403170000.ok - log: output/logs/execute_inference/b30a-0209f019-202403170000.log - jobid: 592 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411240000.ok - log: output/logs/execute_inference/b30a-0209f019-202411240000.log - jobid: 1348 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411240000.ok - wildcards: run_id=b30a-0209f019, init_time=202411240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406090000.ok - log: output/logs/execute_inference/b30a-0209f019-202406090000.log - jobid: 844 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405120000.ok - log: output/logs/execute_inference/b30a-0209f019-202405120000.log - jobid: 760 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405120000.ok - wildcards: run_id=b30a-0209f019, init_time=202405120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403180000.ok - log: output/logs/execute_inference/b30a-0209f019-202403180000.log - jobid: 595 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403180000.ok - wildcards: run_id=b30a-0209f019, init_time=202403180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405130000.ok - log: output/logs/execute_inference/b30a-0209f019-202405130000.log - jobid: 763 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405130000.ok - wildcards: run_id=b30a-0209f019, init_time=202405130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411250000.ok - log: output/logs/execute_inference/b30a-0209f019-202411250000.log - jobid: 1351 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412230000.ok - log: output/logs/execute_inference/b30a-0209f019-202412230000.log - jobid: 1435 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412230000.ok - wildcards: run_id=b30a-0209f019, init_time=202412230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406100000.ok - log: output/logs/execute_inference/b30a-0209f019-202406100000.log - jobid: 847 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406100000.ok - wildcards: run_id=b30a-0209f019, init_time=202406100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411260000.ok - log: output/logs/execute_inference/b30a-0209f019-202411260000.log - jobid: 1354 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411260000.ok - wildcards: run_id=b30a-0209f019, init_time=202411260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403190000.ok - log: output/logs/execute_inference/b30a-0209f019-202403190000.log - jobid: 598 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406110000.ok - log: output/logs/execute_inference/b30a-0209f019-202406110000.log - jobid: 850 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406110000.ok - wildcards: run_id=b30a-0209f019, init_time=202406110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412240000.ok - log: output/logs/execute_inference/b30a-0209f019-202412240000.log - jobid: 1438 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202412240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405140000.ok - log: output/logs/execute_inference/b30a-0209f019-202405140000.log - jobid: 766 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405140000.ok - wildcards: run_id=b30a-0209f019, init_time=202405140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405150000.ok - log: output/logs/execute_inference/b30a-0209f019-202405150000.log - jobid: 769 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405150000.ok - wildcards: run_id=b30a-0209f019, init_time=202405150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412250000.ok - log: output/logs/execute_inference/b30a-0209f019-202412250000.log - jobid: 1441 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202412250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403200000.ok - log: output/logs/execute_inference/b30a-0209f019-202403200000.log - jobid: 601 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411270000.ok - log: output/logs/execute_inference/b30a-0209f019-202411270000.log - jobid: 1357 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406120000.ok - log: output/logs/execute_inference/b30a-0209f019-202406120000.log - jobid: 853 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403210000.ok - log: output/logs/execute_inference/b30a-0209f019-202403210000.log - jobid: 604 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405160000.ok - log: output/logs/execute_inference/b30a-0209f019-202405160000.log - jobid: 772 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406130000.ok - log: output/logs/execute_inference/b30a-0209f019-202406130000.log - jobid: 856 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411280000.ok - log: output/logs/execute_inference/b30a-0209f019-202411280000.log - jobid: 1360 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411280000.ok - wildcards: run_id=b30a-0209f019, init_time=202411280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403220000.ok - log: output/logs/execute_inference/b30a-0209f019-202403220000.log - jobid: 607 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411290000.ok - log: output/logs/execute_inference/b30a-0209f019-202411290000.log - jobid: 1363 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406140000.ok - log: output/logs/execute_inference/b30a-0209f019-202406140000.log - jobid: 859 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406140000.ok - wildcards: run_id=b30a-0209f019, init_time=202406140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405170000.ok - log: output/logs/execute_inference/b30a-0209f019-202405170000.log - jobid: 775 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406150000.ok - log: output/logs/execute_inference/b30a-0209f019-202406150000.log - jobid: 862 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406150000.ok - wildcards: run_id=b30a-0209f019, init_time=202406150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405180000.ok - log: output/logs/execute_inference/b30a-0209f019-202405180000.log - jobid: 778 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405180000.ok - wildcards: run_id=b30a-0209f019, init_time=202405180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403230000.ok - log: output/logs/execute_inference/b30a-0209f019-202403230000.log - jobid: 610 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403230000.ok - wildcards: run_id=b30a-0209f019, init_time=202403230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202411300000.ok - log: output/logs/execute_inference/b30a-0209f019-202411300000.log - jobid: 1366 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202411300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406160000.ok - log: output/logs/execute_inference/b30a-0209f019-202406160000.log - jobid: 865 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406160000.ok - wildcards: run_id=b30a-0209f019, init_time=202406160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405190000.ok - log: output/logs/execute_inference/b30a-0209f019-202405190000.log - jobid: 781 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405190000.ok - wildcards: run_id=b30a-0209f019, init_time=202405190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403240000.ok - log: output/logs/execute_inference/b30a-0209f019-202403240000.log - jobid: 613 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412010000.ok - log: output/logs/execute_inference/b30a-0209f019-202412010000.log - jobid: 1369 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412010000.ok - wildcards: run_id=b30a-0209f019, init_time=202412010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412020000.ok - log: output/logs/execute_inference/b30a-0209f019-202412020000.log - jobid: 1372 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412020000.ok - wildcards: run_id=b30a-0209f019, init_time=202412020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405200000.ok - log: output/logs/execute_inference/b30a-0209f019-202405200000.log - jobid: 784 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405200000.ok - wildcards: run_id=b30a-0209f019, init_time=202405200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406170000.ok - log: output/logs/execute_inference/b30a-0209f019-202406170000.log - jobid: 868 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403250000.ok - log: output/logs/execute_inference/b30a-0209f019-202403250000.log - jobid: 616 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405210000.ok - log: output/logs/execute_inference/b30a-0209f019-202405210000.log - jobid: 787 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405210000.ok - wildcards: run_id=b30a-0209f019, init_time=202405210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406180000.ok - log: output/logs/execute_inference/b30a-0209f019-202406180000.log - jobid: 871 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406180000.ok - wildcards: run_id=b30a-0209f019, init_time=202406180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412030000.ok - log: output/logs/execute_inference/b30a-0209f019-202412030000.log - jobid: 1375 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202412030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403260000.ok - log: output/logs/execute_inference/b30a-0209f019-202403260000.log - jobid: 619 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403260000.ok - wildcards: run_id=b30a-0209f019, init_time=202403260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403270000.ok - log: output/logs/execute_inference/b30a-0209f019-202403270000.log - jobid: 622 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403270000.ok - wildcards: run_id=b30a-0209f019, init_time=202403270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406190000.ok - log: output/logs/execute_inference/b30a-0209f019-202406190000.log - jobid: 874 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405220000.ok - log: output/logs/execute_inference/b30a-0209f019-202405220000.log - jobid: 790 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405220000.ok - wildcards: run_id=b30a-0209f019, init_time=202405220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412040000.ok - log: output/logs/execute_inference/b30a-0209f019-202412040000.log - jobid: 1378 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202412040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412050000.ok - log: output/logs/execute_inference/b30a-0209f019-202412050000.log - jobid: 1381 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412050000.ok - wildcards: run_id=b30a-0209f019, init_time=202412050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406200000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406200000.ok - log: output/logs/execute_inference/b30a-0209f019-202406200000.log - jobid: 877 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406200000.ok - wildcards: run_id=b30a-0209f019, init_time=202406200000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403280000.ok - log: output/logs/execute_inference/b30a-0209f019-202403280000.log - jobid: 625 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405230000.ok - log: output/logs/execute_inference/b30a-0209f019-202405230000.log - jobid: 793 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405230000.ok - wildcards: run_id=b30a-0209f019, init_time=202405230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403290000.ok - log: output/logs/execute_inference/b30a-0209f019-202403290000.log - jobid: 628 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405240000.ok - log: output/logs/execute_inference/b30a-0209f019-202405240000.log - jobid: 796 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405240000.ok - wildcards: run_id=b30a-0209f019, init_time=202405240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412060000.ok - log: output/logs/execute_inference/b30a-0209f019-202412060000.log - jobid: 1384 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412060000.ok - wildcards: run_id=b30a-0209f019, init_time=202412060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406210000.ok - log: output/logs/execute_inference/b30a-0209f019-202406210000.log - jobid: 880 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406210000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406210000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403300000.ok - log: output/logs/execute_inference/b30a-0209f019-202403300000.log - jobid: 631 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202403300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405250000.ok - log: output/logs/execute_inference/b30a-0209f019-202405250000.log - jobid: 799 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405250000.ok - wildcards: run_id=b30a-0209f019, init_time=202405250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412070000.ok - log: output/logs/execute_inference/b30a-0209f019-202412070000.log - jobid: 1387 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412070000.ok - wildcards: run_id=b30a-0209f019, init_time=202412070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406220000.ok - log: output/logs/execute_inference/b30a-0209f019-202406220000.log - jobid: 883 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406220000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406220000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403310000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202403310000.ok - log: output/logs/execute_inference/b30a-0209f019-202403310000.log - jobid: 634 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403310000.ok - wildcards: run_id=b30a-0209f019, init_time=202403310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405260000.ok - log: output/logs/execute_inference/b30a-0209f019-202405260000.log - jobid: 802 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405260000.ok - wildcards: run_id=b30a-0209f019, init_time=202405260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412080000.ok - log: output/logs/execute_inference/b30a-0209f019-202412080000.log - jobid: 1390 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412080000.ok - wildcards: run_id=b30a-0209f019, init_time=202412080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406230000.ok - log: output/logs/execute_inference/b30a-0209f019-202406230000.log - jobid: 886 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406230000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406230000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404010000.ok - log: output/logs/execute_inference/b30a-0209f019-202404010000.log - jobid: 637 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202404010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405270000.ok - log: output/logs/execute_inference/b30a-0209f019-202405270000.log - jobid: 805 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412090000.ok - log: output/logs/execute_inference/b30a-0209f019-202412090000.log - jobid: 1393 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412090000.ok - wildcards: run_id=b30a-0209f019, init_time=202412090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406240000.ok - log: output/logs/execute_inference/b30a-0209f019-202406240000.log - jobid: 889 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406240000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406240000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404020000.ok - log: output/logs/execute_inference/b30a-0209f019-202404020000.log - jobid: 640 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404020000.ok - wildcards: run_id=b30a-0209f019, init_time=202404020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405280000.ok - log: output/logs/execute_inference/b30a-0209f019-202405280000.log - jobid: 808 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412100000.ok - log: output/logs/execute_inference/b30a-0209f019-202412100000.log - jobid: 1396 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202412100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406250000.ok - log: output/logs/execute_inference/b30a-0209f019-202406250000.log - jobid: 892 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406250000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406250000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404030000.ok - log: output/logs/execute_inference/b30a-0209f019-202404030000.log - jobid: 643 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404030000.ok - wildcards: run_id=b30a-0209f019, init_time=202404030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405290000.ok - log: output/logs/execute_inference/b30a-0209f019-202405290000.log - jobid: 811 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405290000.ok - wildcards: run_id=b30a-0209f019, init_time=202405290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412110000.ok - log: output/logs/execute_inference/b30a-0209f019-202412110000.log - jobid: 1399 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202412110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406260000.ok - log: output/logs/execute_inference/b30a-0209f019-202406260000.log - jobid: 895 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406260000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406260000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404040000.ok - log: output/logs/execute_inference/b30a-0209f019-202404040000.log - jobid: 646 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202404040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405300000.ok - log: output/logs/execute_inference/b30a-0209f019-202405300000.log - jobid: 814 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202405300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412120000.ok - log: output/logs/execute_inference/b30a-0209f019-202412120000.log - jobid: 1402 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412120000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202412120000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406270000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406270000.ok - log: output/logs/execute_inference/b30a-0209f019-202406270000.log - jobid: 898 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406270000.ok - wildcards: run_id=b30a-0209f019, init_time=202406270000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404050000.ok - log: output/logs/execute_inference/b30a-0209f019-202404050000.log - jobid: 649 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404050000.ok - wildcards: run_id=b30a-0209f019, init_time=202404050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405310000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202405310000.ok - log: output/logs/execute_inference/b30a-0209f019-202405310000.log - jobid: 817 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405310000.ok - wildcards: run_id=b30a-0209f019, init_time=202405310000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412130000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412130000.ok - log: output/logs/execute_inference/b30a-0209f019-202412130000.log - jobid: 1405 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412130000.ok - wildcards: run_id=b30a-0209f019, init_time=202412130000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406280000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406280000.ok - log: output/logs/execute_inference/b30a-0209f019-202406280000.log - jobid: 901 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406280000.ok - wildcards: run_id=b30a-0209f019, init_time=202406280000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404060000.ok - log: output/logs/execute_inference/b30a-0209f019-202404060000.log - jobid: 652 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202404060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406010000.ok - log: output/logs/execute_inference/b30a-0209f019-202406010000.log - jobid: 820 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406010000.ok - wildcards: run_id=b30a-0209f019, init_time=202406010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412140000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412140000.ok - log: output/logs/execute_inference/b30a-0209f019-202412140000.log - jobid: 1408 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412140000.ok - wildcards: run_id=b30a-0209f019, init_time=202412140000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406290000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406290000.ok - log: output/logs/execute_inference/b30a-0209f019-202406290000.log - jobid: 904 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406290000.ok - wildcards: run_id=b30a-0209f019, init_time=202406290000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404070000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404070000.ok - log: output/logs/execute_inference/b30a-0209f019-202404070000.log - jobid: 655 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404070000.ok - wildcards: run_id=b30a-0209f019, init_time=202404070000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406020000.ok - log: output/logs/execute_inference/b30a-0209f019-202406020000.log - jobid: 823 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412150000.ok - log: output/logs/execute_inference/b30a-0209f019-202412150000.log - jobid: 1411 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412150000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202412150000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406300000.ok - log: output/logs/execute_inference/b30a-0209f019-202406300000.log - jobid: 907 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406300000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406300000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404080000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404080000.ok - log: output/logs/execute_inference/b30a-0209f019-202404080000.log - jobid: 658 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404080000.ok - wildcards: run_id=b30a-0209f019, init_time=202404080000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406030000.ok - log: output/logs/execute_inference/b30a-0209f019-202406030000.log - jobid: 826 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412160000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412160000.ok - log: output/logs/execute_inference/b30a-0209f019-202412160000.log - jobid: 1414 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412160000.ok - wildcards: run_id=b30a-0209f019, init_time=202412160000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407010000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407010000.ok - log: output/logs/execute_inference/b30a-0209f019-202407010000.log - jobid: 910 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407010000.ok - wildcards: run_id=b30a-0209f019, init_time=202407010000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404090000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404090000.ok - log: output/logs/execute_inference/b30a-0209f019-202404090000.log - jobid: 661 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404090000.ok - wildcards: run_id=b30a-0209f019, init_time=202404090000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406040000.ok - log: output/logs/execute_inference/b30a-0209f019-202406040000.log - jobid: 829 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412170000.ok - log: output/logs/execute_inference/b30a-0209f019-202412170000.log - jobid: 1417 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412170000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202412170000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407020000.ok - log: output/logs/execute_inference/b30a-0209f019-202407020000.log - jobid: 913 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407020000.ok - wildcards: run_id=b30a-0209f019, init_time=202407020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406050000.ok - log: output/logs/execute_inference/b30a-0209f019-202406050000.log - jobid: 832 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406050000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202406050000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404100000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404100000.ok - log: output/logs/execute_inference/b30a-0209f019-202404100000.log - jobid: 664 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404100000.ok - wildcards: run_id=b30a-0209f019, init_time=202404100000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407030000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407030000.ok - log: output/logs/execute_inference/b30a-0209f019-202407030000.log - jobid: 916 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407030000.ok - wildcards: run_id=b30a-0209f019, init_time=202407030000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412180000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412180000.ok - log: output/logs/execute_inference/b30a-0209f019-202412180000.log - jobid: 1420 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412180000.ok - wildcards: run_id=b30a-0209f019, init_time=202412180000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401020000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202401020000.ok - log: output/logs/execute_inference/b30a-0209f019-202401020000.log - jobid: 364 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202401020000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412190000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202412190000.ok - log: output/logs/execute_inference/b30a-0209f019-202412190000.log - jobid: 1423 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412190000.ok - wildcards: run_id=b30a-0209f019, init_time=202412190000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406060000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202406060000.ok - log: output/logs/execute_inference/b30a-0209f019-202406060000.log - jobid: 835 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406060000.ok - wildcards: run_id=b30a-0209f019, init_time=202406060000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202404110000.ok - log: output/logs/execute_inference/b30a-0209f019-202404110000.log - jobid: 667 - reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404110000.ok, output/data/runs/b30a-0209f019/venv.squashfs - wildcards: run_id=b30a-0209f019, init_time=202404110000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -[Thu Feb 26 12:18:24 2026] -rule execute_inference: - input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407040000.ok, output/data/runs/b30a-0209f019/venv.squashfs - output: output/logs/execute_inference/b30a-0209f019-202407040000.ok - log: output/logs/execute_inference/b30a-0209f019-202407040000.log - jobid: 919 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407040000.ok - wildcards: run_id=b30a-0209f019, init_time=202407040000 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc -Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc - -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408140000.log - jobid: 1041 - reason: Missing output files: output/data/runs/b30a-0209f019/202408140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408140000.ok - wildcards: run_id=b30a-0209f019, init_time=202408140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410090000.log - jobid: 1209 - reason: Missing output files: output/data/runs/b30a-0209f019/202410090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410090000.ok - wildcards: run_id=b30a-0209f019, init_time=202410090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404240000.log - jobid: 705 - reason: Missing output files: output/data/runs/b30a-0209f019/202404240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404240000.ok - wildcards: run_id=b30a-0209f019, init_time=202404240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401030000.log - jobid: 369 - reason: Missing output files: output/data/runs/b30a-0209f019/202401030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401030000.ok - wildcards: run_id=b30a-0209f019, init_time=202401030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406190000.log - jobid: 873 - reason: Missing output files: output/data/runs/b30a-0209f019/202406190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406190000.ok - wildcards: run_id=b30a-0209f019, init_time=202406190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412040000.log - jobid: 1377 - reason: Missing output files: output/data/runs/b30a-0209f019/202412040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412040000.ok - wildcards: run_id=b30a-0209f019, init_time=202412040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412050000.log - jobid: 1380 - reason: Missing output files: output/data/runs/b30a-0209f019/202412050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412050000.ok - wildcards: run_id=b30a-0209f019, init_time=202412050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410100000.log - jobid: 1212 - reason: Missing output files: output/data/runs/b30a-0209f019/202410100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410100000.ok - wildcards: run_id=b30a-0209f019, init_time=202410100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404250000.log - jobid: 708 - reason: Missing output files: output/data/runs/b30a-0209f019/202404250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404250000.ok - wildcards: run_id=b30a-0209f019, init_time=202404250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406200000.log - jobid: 876 - reason: Missing output files: output/data/runs/b30a-0209f019/202406200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406200000.ok - wildcards: run_id=b30a-0209f019, init_time=202406200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401040000.log - jobid: 372 - reason: Missing output files: output/data/runs/b30a-0209f019/202401040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401040000.ok - wildcards: run_id=b30a-0209f019, init_time=202401040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408150000.log - jobid: 1044 - reason: Missing output files: output/data/runs/b30a-0209f019/202408150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408150000.ok - wildcards: run_id=b30a-0209f019, init_time=202408150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408160000.log - jobid: 1047 - reason: Missing output files: output/data/runs/b30a-0209f019/202408160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408160000.ok - wildcards: run_id=b30a-0209f019, init_time=202408160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401050000.log - jobid: 375 - reason: Missing output files: output/data/runs/b30a-0209f019/202401050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401050000.ok - wildcards: run_id=b30a-0209f019, init_time=202401050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404260000.log - jobid: 711 - reason: Missing output files: output/data/runs/b30a-0209f019/202404260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404260000.ok - wildcards: run_id=b30a-0209f019, init_time=202404260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410110000.log - jobid: 1215 - reason: Missing output files: output/data/runs/b30a-0209f019/202410110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410110000.ok - wildcards: run_id=b30a-0209f019, init_time=202410110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412060000.log - jobid: 1383 - reason: Missing output files: output/data/runs/b30a-0209f019/202412060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412060000.ok - wildcards: run_id=b30a-0209f019, init_time=202412060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406210000.log - jobid: 879 - reason: Missing output files: output/data/runs/b30a-0209f019/202406210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406210000.ok - wildcards: run_id=b30a-0209f019, init_time=202406210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406220000.log - jobid: 882 - reason: Missing output files: output/data/runs/b30a-0209f019/202406220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406220000.ok - wildcards: run_id=b30a-0209f019, init_time=202406220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412070000.log - jobid: 1386 - reason: Missing output files: output/data/runs/b30a-0209f019/202412070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412070000.ok - wildcards: run_id=b30a-0209f019, init_time=202412070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410120000.log - jobid: 1218 - reason: Missing output files: output/data/runs/b30a-0209f019/202410120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410120000.ok - wildcards: run_id=b30a-0209f019, init_time=202410120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401060000.log - jobid: 378 - reason: Missing output files: output/data/runs/b30a-0209f019/202401060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401060000.ok - wildcards: run_id=b30a-0209f019, init_time=202401060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404270000.log - jobid: 714 - reason: Missing output files: output/data/runs/b30a-0209f019/202404270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404270000.ok - wildcards: run_id=b30a-0209f019, init_time=202404270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408170000.log - jobid: 1050 - reason: Missing output files: output/data/runs/b30a-0209f019/202408170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408170000.ok - wildcards: run_id=b30a-0209f019, init_time=202408170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408180000.log - jobid: 1053 - reason: Missing output files: output/data/runs/b30a-0209f019/202408180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408180000.ok - wildcards: run_id=b30a-0209f019, init_time=202408180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404280000.log - jobid: 717 - reason: Missing output files: output/data/runs/b30a-0209f019/202404280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404280000.ok - wildcards: run_id=b30a-0209f019, init_time=202404280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410130000.log - jobid: 1221 - reason: Missing output files: output/data/runs/b30a-0209f019/202410130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410130000.ok - wildcards: run_id=b30a-0209f019, init_time=202410130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401070000.log - jobid: 381 - reason: Missing output files: output/data/runs/b30a-0209f019/202401070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401070000.ok - wildcards: run_id=b30a-0209f019, init_time=202401070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412080000.log - jobid: 1389 - reason: Missing output files: output/data/runs/b30a-0209f019/202412080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412080000.ok - wildcards: run_id=b30a-0209f019, init_time=202412080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406230000.log - jobid: 885 - reason: Missing output files: output/data/runs/b30a-0209f019/202406230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406230000.ok - wildcards: run_id=b30a-0209f019, init_time=202406230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406240000.log - jobid: 888 - reason: Missing output files: output/data/runs/b30a-0209f019/202406240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406240000.ok - wildcards: run_id=b30a-0209f019, init_time=202406240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412090000.log - jobid: 1392 - reason: Missing output files: output/data/runs/b30a-0209f019/202412090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412090000.ok - wildcards: run_id=b30a-0209f019, init_time=202412090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401080000.log - jobid: 384 - reason: Missing output files: output/data/runs/b30a-0209f019/202401080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401080000.ok - wildcards: run_id=b30a-0209f019, init_time=202401080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410140000.log - jobid: 1224 - reason: Missing output files: output/data/runs/b30a-0209f019/202410140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410140000.ok - wildcards: run_id=b30a-0209f019, init_time=202410140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404290000.log - jobid: 720 - reason: Missing output files: output/data/runs/b30a-0209f019/202404290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404290000.ok - wildcards: run_id=b30a-0209f019, init_time=202404290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408190000.log - jobid: 1056 - reason: Missing output files: output/data/runs/b30a-0209f019/202408190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408190000.ok - wildcards: run_id=b30a-0209f019, init_time=202408190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408200000.log - jobid: 1059 - reason: Missing output files: output/data/runs/b30a-0209f019/202408200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408200000.ok - wildcards: run_id=b30a-0209f019, init_time=202408200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404300000.log - jobid: 723 - reason: Missing output files: output/data/runs/b30a-0209f019/202404300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404300000.ok - wildcards: run_id=b30a-0209f019, init_time=202404300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410150000.log - jobid: 1227 - reason: Missing output files: output/data/runs/b30a-0209f019/202410150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410150000.ok - wildcards: run_id=b30a-0209f019, init_time=202410150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401090000.log - jobid: 387 - reason: Missing output files: output/data/runs/b30a-0209f019/202401090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401090000.ok - wildcards: run_id=b30a-0209f019, init_time=202401090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412100000.log - jobid: 1395 - reason: Missing output files: output/data/runs/b30a-0209f019/202412100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412100000.ok - wildcards: run_id=b30a-0209f019, init_time=202412100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406250000.log - jobid: 891 - reason: Missing output files: output/data/runs/b30a-0209f019/202406250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406250000.ok - wildcards: run_id=b30a-0209f019, init_time=202406250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406260000.log - jobid: 894 - reason: Missing output files: output/data/runs/b30a-0209f019/202406260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406260000.ok - wildcards: run_id=b30a-0209f019, init_time=202406260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412110000.log - jobid: 1398 - reason: Missing output files: output/data/runs/b30a-0209f019/202412110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412110000.ok - wildcards: run_id=b30a-0209f019, init_time=202412110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401100000.log - jobid: 390 - reason: Missing output files: output/data/runs/b30a-0209f019/202401100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401100000.ok - wildcards: run_id=b30a-0209f019, init_time=202401100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410160000.log - jobid: 1230 - reason: Missing output files: output/data/runs/b30a-0209f019/202410160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410160000.ok - wildcards: run_id=b30a-0209f019, init_time=202410160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408210000.log - jobid: 1062 - reason: Missing output files: output/data/runs/b30a-0209f019/202408210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408210000.ok - wildcards: run_id=b30a-0209f019, init_time=202408210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405010000.log - jobid: 726 - reason: Missing output files: output/data/runs/b30a-0209f019/202405010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405010000.ok - wildcards: run_id=b30a-0209f019, init_time=202405010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408220000.log - jobid: 1065 - reason: Missing output files: output/data/runs/b30a-0209f019/202408220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408220000.ok - wildcards: run_id=b30a-0209f019, init_time=202408220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405020000.log - jobid: 729 - reason: Missing output files: output/data/runs/b30a-0209f019/202405020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405020000.ok - wildcards: run_id=b30a-0209f019, init_time=202405020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410170000.log - jobid: 1233 - reason: Missing output files: output/data/runs/b30a-0209f019/202410170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410170000.ok - wildcards: run_id=b30a-0209f019, init_time=202410170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401110000.log - jobid: 393 - reason: Missing output files: output/data/runs/b30a-0209f019/202401110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401110000.ok - wildcards: run_id=b30a-0209f019, init_time=202401110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412120000.log - jobid: 1401 - reason: Missing output files: output/data/runs/b30a-0209f019/202412120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412120000.ok - wildcards: run_id=b30a-0209f019, init_time=202412120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406270000.log - jobid: 897 - reason: Missing output files: output/data/runs/b30a-0209f019/202406270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406270000.ok - wildcards: run_id=b30a-0209f019, init_time=202406270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406280000.log - jobid: 900 - reason: Missing output files: output/data/runs/b30a-0209f019/202406280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406280000.ok - wildcards: run_id=b30a-0209f019, init_time=202406280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412130000.log - jobid: 1404 - reason: Missing output files: output/data/runs/b30a-0209f019/202412130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412130000.ok - wildcards: run_id=b30a-0209f019, init_time=202412130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405030000.log - jobid: 732 - reason: Missing output files: output/data/runs/b30a-0209f019/202405030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405030000.ok - wildcards: run_id=b30a-0209f019, init_time=202405030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408230000.log - jobid: 1068 - reason: Missing output files: output/data/runs/b30a-0209f019/202408230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408230000.ok - wildcards: run_id=b30a-0209f019, init_time=202408230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410180000.log - jobid: 1236 - reason: Missing output files: output/data/runs/b30a-0209f019/202410180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410180000.ok - wildcards: run_id=b30a-0209f019, init_time=202410180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401120000.log - jobid: 396 - reason: Missing output files: output/data/runs/b30a-0209f019/202401120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401120000.ok - wildcards: run_id=b30a-0209f019, init_time=202401120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412140000.log - jobid: 1407 - reason: Missing output files: output/data/runs/b30a-0209f019/202412140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412140000.ok - wildcards: run_id=b30a-0209f019, init_time=202412140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406290000.log - jobid: 903 - reason: Missing output files: output/data/runs/b30a-0209f019/202406290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406290000.ok - wildcards: run_id=b30a-0209f019, init_time=202406290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401130000.log - jobid: 399 - reason: Missing output files: output/data/runs/b30a-0209f019/202401130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401130000.ok - wildcards: run_id=b30a-0209f019, init_time=202401130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408240000.log - jobid: 1071 - reason: Missing output files: output/data/runs/b30a-0209f019/202408240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408240000.ok - wildcards: run_id=b30a-0209f019, init_time=202408240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405040000.log - jobid: 735 - reason: Missing output files: output/data/runs/b30a-0209f019/202405040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405040000.ok - wildcards: run_id=b30a-0209f019, init_time=202405040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410190000.log - jobid: 1239 - reason: Missing output files: output/data/runs/b30a-0209f019/202410190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410190000.ok - wildcards: run_id=b30a-0209f019, init_time=202410190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401140000.log - jobid: 402 - reason: Missing output files: output/data/runs/b30a-0209f019/202401140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401140000.ok - wildcards: run_id=b30a-0209f019, init_time=202401140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405050000.log - jobid: 738 - reason: Missing output files: output/data/runs/b30a-0209f019/202405050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405050000.ok - wildcards: run_id=b30a-0209f019, init_time=202405050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406300000.log - jobid: 906 - reason: Missing output files: output/data/runs/b30a-0209f019/202406300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406300000.ok - wildcards: run_id=b30a-0209f019, init_time=202406300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412150000.log - jobid: 1410 - reason: Missing output files: output/data/runs/b30a-0209f019/202412150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412150000.ok - wildcards: run_id=b30a-0209f019, init_time=202412150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410200000.log - jobid: 1242 - reason: Missing output files: output/data/runs/b30a-0209f019/202410200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410200000.ok - wildcards: run_id=b30a-0209f019, init_time=202410200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408250000.log - jobid: 1074 - reason: Missing output files: output/data/runs/b30a-0209f019/202408250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408250000.ok - wildcards: run_id=b30a-0209f019, init_time=202408250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412160000.log - jobid: 1413 - reason: Missing output files: output/data/runs/b30a-0209f019/202412160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412160000.ok - wildcards: run_id=b30a-0209f019, init_time=202412160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410210000.log - jobid: 1245 - reason: Missing output files: output/data/runs/b30a-0209f019/202410210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410210000.ok - wildcards: run_id=b30a-0209f019, init_time=202410210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408260000.log - jobid: 1077 - reason: Missing output files: output/data/runs/b30a-0209f019/202408260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408260000.ok - wildcards: run_id=b30a-0209f019, init_time=202408260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405060000.log - jobid: 741 - reason: Missing output files: output/data/runs/b30a-0209f019/202405060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405060000.ok - wildcards: run_id=b30a-0209f019, init_time=202405060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401150000.log - jobid: 405 - reason: Missing output files: output/data/runs/b30a-0209f019/202401150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401150000.ok - wildcards: run_id=b30a-0209f019, init_time=202401150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407010000.log - jobid: 909 - reason: Missing output files: output/data/runs/b30a-0209f019/202407010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407010000.ok - wildcards: run_id=b30a-0209f019, init_time=202407010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407020000.log - jobid: 912 - reason: Missing output files: output/data/runs/b30a-0209f019/202407020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407020000.ok - wildcards: run_id=b30a-0209f019, init_time=202407020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412170000.log - jobid: 1416 - reason: Missing output files: output/data/runs/b30a-0209f019/202412170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412170000.ok - wildcards: run_id=b30a-0209f019, init_time=202412170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401160000.log - jobid: 408 - reason: Missing output files: output/data/runs/b30a-0209f019/202401160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401160000.ok - wildcards: run_id=b30a-0209f019, init_time=202401160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405070000.log - jobid: 744 - reason: Missing output files: output/data/runs/b30a-0209f019/202405070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405070000.ok - wildcards: run_id=b30a-0209f019, init_time=202405070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408270000.log - jobid: 1080 - reason: Missing output files: output/data/runs/b30a-0209f019/202408270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408270000.ok - wildcards: run_id=b30a-0209f019, init_time=202408270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410220000.log - jobid: 1248 - reason: Missing output files: output/data/runs/b30a-0209f019/202410220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410220000.ok - wildcards: run_id=b30a-0209f019, init_time=202410220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410230000.log - jobid: 1251 - reason: Missing output files: output/data/runs/b30a-0209f019/202410230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410230000.ok - wildcards: run_id=b30a-0209f019, init_time=202410230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405080000.log - jobid: 747 - reason: Missing output files: output/data/runs/b30a-0209f019/202405080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405080000.ok - wildcards: run_id=b30a-0209f019, init_time=202405080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401170000.log - jobid: 411 - reason: Missing output files: output/data/runs/b30a-0209f019/202401170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401170000.ok - wildcards: run_id=b30a-0209f019, init_time=202401170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408280000.log - jobid: 1083 - reason: Missing output files: output/data/runs/b30a-0209f019/202408280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408280000.ok - wildcards: run_id=b30a-0209f019, init_time=202408280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407030000.log - jobid: 915 - reason: Missing output files: output/data/runs/b30a-0209f019/202407030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407030000.ok - wildcards: run_id=b30a-0209f019, init_time=202407030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412180000.log - jobid: 1419 - reason: Missing output files: output/data/runs/b30a-0209f019/202412180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412180000.ok - wildcards: run_id=b30a-0209f019, init_time=202412180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407040000.log - jobid: 918 - reason: Missing output files: output/data/runs/b30a-0209f019/202407040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407040000.ok - wildcards: run_id=b30a-0209f019, init_time=202407040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412190000.log - jobid: 1422 - reason: Missing output files: output/data/runs/b30a-0209f019/202412190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412190000.ok - wildcards: run_id=b30a-0209f019, init_time=202412190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408290000.log - jobid: 1086 - reason: Missing output files: output/data/runs/b30a-0209f019/202408290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408290000.ok - wildcards: run_id=b30a-0209f019, init_time=202408290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401180000.log - jobid: 414 - reason: Missing output files: output/data/runs/b30a-0209f019/202401180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401180000.ok - wildcards: run_id=b30a-0209f019, init_time=202401180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405090000.log - jobid: 750 - reason: Missing output files: output/data/runs/b30a-0209f019/202405090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405090000.ok - wildcards: run_id=b30a-0209f019, init_time=202405090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410240000.log - jobid: 1254 - reason: Missing output files: output/data/runs/b30a-0209f019/202410240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410240000.ok - wildcards: run_id=b30a-0209f019, init_time=202410240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401190000.log - jobid: 417 - reason: Missing output files: output/data/runs/b30a-0209f019/202401190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401190000.ok - wildcards: run_id=b30a-0209f019, init_time=202401190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407050000.log - jobid: 921 - reason: Missing output files: output/data/runs/b30a-0209f019/202407050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407050000.ok - wildcards: run_id=b30a-0209f019, init_time=202407050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408300000.log - jobid: 1089 - reason: Missing output files: output/data/runs/b30a-0209f019/202408300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408300000.ok - wildcards: run_id=b30a-0209f019, init_time=202408300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410250000.log - jobid: 1257 - reason: Missing output files: output/data/runs/b30a-0209f019/202410250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410250000.ok - wildcards: run_id=b30a-0209f019, init_time=202410250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405100000.log - jobid: 753 - reason: Missing output files: output/data/runs/b30a-0209f019/202405100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405100000.ok - wildcards: run_id=b30a-0209f019, init_time=202405100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412200000.log - jobid: 1425 - reason: Missing output files: output/data/runs/b30a-0209f019/202412200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412200000.ok - wildcards: run_id=b30a-0209f019, init_time=202412200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412210000.log - jobid: 1428 - reason: Missing output files: output/data/runs/b30a-0209f019/202412210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412210000.ok - wildcards: run_id=b30a-0209f019, init_time=202412210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405110000.log - jobid: 756 - reason: Missing output files: output/data/runs/b30a-0209f019/202405110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405110000.ok - wildcards: run_id=b30a-0209f019, init_time=202405110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410260000.log - jobid: 1260 - reason: Missing output files: output/data/runs/b30a-0209f019/202410260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410260000.ok - wildcards: run_id=b30a-0209f019, init_time=202410260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407060000.log - jobid: 924 - reason: Missing output files: output/data/runs/b30a-0209f019/202407060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407060000.ok - wildcards: run_id=b30a-0209f019, init_time=202407060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401200000.log - jobid: 420 - reason: Missing output files: output/data/runs/b30a-0209f019/202401200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401200000.ok - wildcards: run_id=b30a-0209f019, init_time=202401200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408310000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408310000.log - jobid: 1092 - reason: Missing output files: output/data/runs/b30a-0209f019/202408310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408310000.ok - wildcards: run_id=b30a-0209f019, init_time=202408310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401210000.log - jobid: 423 - reason: Missing output files: output/data/runs/b30a-0209f019/202401210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401210000.ok - wildcards: run_id=b30a-0209f019, init_time=202401210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407070000.log - jobid: 927 - reason: Missing output files: output/data/runs/b30a-0209f019/202407070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407070000.ok - wildcards: run_id=b30a-0209f019, init_time=202407070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409010000.log - jobid: 1095 - reason: Missing output files: output/data/runs/b30a-0209f019/202409010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409010000.ok - wildcards: run_id=b30a-0209f019, init_time=202409010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410270000.log - jobid: 1263 - reason: Missing output files: output/data/runs/b30a-0209f019/202410270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410270000.ok - wildcards: run_id=b30a-0209f019, init_time=202410270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412220000.log - jobid: 1431 - reason: Missing output files: output/data/runs/b30a-0209f019/202412220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412220000.ok - wildcards: run_id=b30a-0209f019, init_time=202412220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405120000.log - jobid: 759 - reason: Missing output files: output/data/runs/b30a-0209f019/202405120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405120000.ok - wildcards: run_id=b30a-0209f019, init_time=202405120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412230000.log - jobid: 1434 - reason: Missing output files: output/data/runs/b30a-0209f019/202412230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412230000.ok - wildcards: run_id=b30a-0209f019, init_time=202412230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405130000.log - jobid: 762 - reason: Missing output files: output/data/runs/b30a-0209f019/202405130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405130000.ok - wildcards: run_id=b30a-0209f019, init_time=202405130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_all_6.png - jobid: 1455 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_all_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=all, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410280000.log - jobid: 1266 - reason: Missing output files: output/data/runs/b30a-0209f019/202410280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410280000.ok - wildcards: run_id=b30a-0209f019, init_time=202410280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409020000.log - jobid: 1098 - reason: Missing output files: output/data/runs/b30a-0209f019/202409020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409020000.ok - wildcards: run_id=b30a-0209f019, init_time=202409020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407080000.log - jobid: 930 - reason: Missing output files: output/data/runs/b30a-0209f019/202407080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407080000.ok - wildcards: run_id=b30a-0209f019, init_time=202407080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401220000.log - jobid: 426 - reason: Missing output files: output/data/runs/b30a-0209f019/202401220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401220000.ok - wildcards: run_id=b30a-0209f019, init_time=202401220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401230000.log - jobid: 429 - reason: Missing output files: output/data/runs/b30a-0209f019/202401230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401230000.ok - wildcards: run_id=b30a-0209f019, init_time=202401230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407090000.log - jobid: 933 - reason: Missing output files: output/data/runs/b30a-0209f019/202407090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407090000.ok - wildcards: run_id=b30a-0209f019, init_time=202407090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409030000.log - jobid: 1101 - reason: Missing output files: output/data/runs/b30a-0209f019/202409030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409030000.ok - wildcards: run_id=b30a-0209f019, init_time=202409030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410290000.log - jobid: 1269 - reason: Missing output files: output/data/runs/b30a-0209f019/202410290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410290000.ok - wildcards: run_id=b30a-0209f019, init_time=202410290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_DJF_6.png - jobid: 1456 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_DJF_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=DJF, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412240000.log - jobid: 1437 - reason: Missing output files: output/data/runs/b30a-0209f019/202412240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412240000.ok - wildcards: run_id=b30a-0209f019, init_time=202412240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405140000.log - jobid: 765 - reason: Missing output files: output/data/runs/b30a-0209f019/202405140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405140000.ok - wildcards: run_id=b30a-0209f019, init_time=202405140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412250000.log - jobid: 1440 - reason: Missing output files: output/data/runs/b30a-0209f019/202412250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412250000.ok - wildcards: run_id=b30a-0209f019, init_time=202412250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405150000.log - jobid: 768 - reason: Missing output files: output/data/runs/b30a-0209f019/202405150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405150000.ok - wildcards: run_id=b30a-0209f019, init_time=202405150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_MAM_6.png - jobid: 1457 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_MAM_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=MAM, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410300000.log - jobid: 1272 - reason: Missing output files: output/data/runs/b30a-0209f019/202410300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410300000.ok - wildcards: run_id=b30a-0209f019, init_time=202410300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409040000.log - jobid: 1104 - reason: Missing output files: output/data/runs/b30a-0209f019/202409040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409040000.ok - wildcards: run_id=b30a-0209f019, init_time=202409040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407100000.log - jobid: 936 - reason: Missing output files: output/data/runs/b30a-0209f019/202407100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407100000.ok - wildcards: run_id=b30a-0209f019, init_time=202407100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401240000.log - jobid: 432 - reason: Missing output files: output/data/runs/b30a-0209f019/202401240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401240000.ok - wildcards: run_id=b30a-0209f019, init_time=202401240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401250000.log - jobid: 435 - reason: Missing output files: output/data/runs/b30a-0209f019/202401250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401250000.ok - wildcards: run_id=b30a-0209f019, init_time=202401250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407110000.log - jobid: 939 - reason: Missing output files: output/data/runs/b30a-0209f019/202407110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407110000.ok - wildcards: run_id=b30a-0209f019, init_time=202407110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409050000.log - jobid: 1107 - reason: Missing output files: output/data/runs/b30a-0209f019/202409050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409050000.ok - wildcards: run_id=b30a-0209f019, init_time=202409050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410310000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410310000.log - jobid: 1275 - reason: Missing output files: output/data/runs/b30a-0209f019/202410310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410310000.ok - wildcards: run_id=b30a-0209f019, init_time=202410310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_JJA_6.png - jobid: 1458 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_JJA_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=JJA, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405160000.log - jobid: 771 - reason: Missing output files: output/data/runs/b30a-0209f019/202405160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405160000.ok - wildcards: run_id=b30a-0209f019, init_time=202405160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405170000.log - jobid: 774 - reason: Missing output files: output/data/runs/b30a-0209f019/202405170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405170000.ok - wildcards: run_id=b30a-0209f019, init_time=202405170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_SON_6.png - jobid: 1459 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_SON_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=SON, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409060000.log - jobid: 1110 - reason: Missing output files: output/data/runs/b30a-0209f019/202409060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409060000.ok - wildcards: run_id=b30a-0209f019, init_time=202409060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411010000.log - jobid: 1278 - reason: Missing output files: output/data/runs/b30a-0209f019/202411010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411010000.ok - wildcards: run_id=b30a-0209f019, init_time=202411010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407120000.log - jobid: 942 - reason: Missing output files: output/data/runs/b30a-0209f019/202407120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407120000.ok - wildcards: run_id=b30a-0209f019, init_time=202407120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401260000.log - jobid: 438 - reason: Missing output files: output/data/runs/b30a-0209f019/202401260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401260000.ok - wildcards: run_id=b30a-0209f019, init_time=202401260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407130000.log - jobid: 945 - reason: Missing output files: output/data/runs/b30a-0209f019/202407130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407130000.ok - wildcards: run_id=b30a-0209f019, init_time=202407130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409070000.log - jobid: 1113 - reason: Missing output files: output/data/runs/b30a-0209f019/202409070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409070000.ok - wildcards: run_id=b30a-0209f019, init_time=202409070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401270000.log - jobid: 441 - reason: Missing output files: output/data/runs/b30a-0209f019/202401270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401270000.ok - wildcards: run_id=b30a-0209f019, init_time=202401270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411020000.log - jobid: 1281 - reason: Missing output files: output/data/runs/b30a-0209f019/202411020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411020000.ok - wildcards: run_id=b30a-0209f019, init_time=202411020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_all_6.png - jobid: 1460 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_all_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=all, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405180000.log - jobid: 777 - reason: Missing output files: output/data/runs/b30a-0209f019/202405180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405180000.ok - wildcards: run_id=b30a-0209f019, init_time=202405180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405190000.log - jobid: 780 - reason: Missing output files: output/data/runs/b30a-0209f019/202405190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405190000.ok - wildcards: run_id=b30a-0209f019, init_time=202405190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_DJF_6.png - jobid: 1461 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_DJF_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=DJF, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411030000.log - jobid: 1284 - reason: Missing output files: output/data/runs/b30a-0209f019/202411030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411030000.ok - wildcards: run_id=b30a-0209f019, init_time=202411030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407140000.log - jobid: 948 - reason: Missing output files: output/data/runs/b30a-0209f019/202407140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407140000.ok - wildcards: run_id=b30a-0209f019, init_time=202407140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409080000.log - jobid: 1116 - reason: Missing output files: output/data/runs/b30a-0209f019/202409080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409080000.ok - wildcards: run_id=b30a-0209f019, init_time=202409080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401280000.log - jobid: 444 - reason: Missing output files: output/data/runs/b30a-0209f019/202401280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401280000.ok - wildcards: run_id=b30a-0209f019, init_time=202401280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401290000.log - jobid: 447 - reason: Missing output files: output/data/runs/b30a-0209f019/202401290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401290000.ok - wildcards: run_id=b30a-0209f019, init_time=202401290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409090000.log - jobid: 1119 - reason: Missing output files: output/data/runs/b30a-0209f019/202409090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409090000.ok - wildcards: run_id=b30a-0209f019, init_time=202409090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407150000.log - jobid: 951 - reason: Missing output files: output/data/runs/b30a-0209f019/202407150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407150000.ok - wildcards: run_id=b30a-0209f019, init_time=202407150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411040000.log - jobid: 1287 - reason: Missing output files: output/data/runs/b30a-0209f019/202411040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411040000.ok - wildcards: run_id=b30a-0209f019, init_time=202411040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_MAM_6.png - jobid: 1462 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_MAM_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=MAM, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405200000.log - jobid: 783 - reason: Missing output files: output/data/runs/b30a-0209f019/202405200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405200000.ok - wildcards: run_id=b30a-0209f019, init_time=202405200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405210000.log - jobid: 786 - reason: Missing output files: output/data/runs/b30a-0209f019/202405210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405210000.ok - wildcards: run_id=b30a-0209f019, init_time=202405210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_JJA_6.png - jobid: 1463 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_JJA_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=JJA, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411050000.log - jobid: 1290 - reason: Missing output files: output/data/runs/b30a-0209f019/202411050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411050000.ok - wildcards: run_id=b30a-0209f019, init_time=202411050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407160000.log - jobid: 954 - reason: Missing output files: output/data/runs/b30a-0209f019/202407160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407160000.ok - wildcards: run_id=b30a-0209f019, init_time=202407160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409100000.log - jobid: 1122 - reason: Missing output files: output/data/runs/b30a-0209f019/202409100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409100000.ok - wildcards: run_id=b30a-0209f019, init_time=202409100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401300000.log - jobid: 450 - reason: Missing output files: output/data/runs/b30a-0209f019/202401300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401300000.ok - wildcards: run_id=b30a-0209f019, init_time=202401300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401310000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401310000.log - jobid: 453 - reason: Missing output files: output/data/runs/b30a-0209f019/202401310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401310000.ok - wildcards: run_id=b30a-0209f019, init_time=202401310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps_baseline: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_SON_6.png - jobid: 1464 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_SON_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=SON, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411060000.log - jobid: 1293 - reason: Missing output files: output/data/runs/b30a-0209f019/202411060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411060000.ok - wildcards: run_id=b30a-0209f019, init_time=202411060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409110000.log - jobid: 1125 - reason: Missing output files: output/data/runs/b30a-0209f019/202409110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409110000.ok - wildcards: run_id=b30a-0209f019, init_time=202409110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405220000.log - jobid: 789 - reason: Missing output files: output/data/runs/b30a-0209f019/202405220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405220000.ok - wildcards: run_id=b30a-0209f019, init_time=202405220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407170000.log - jobid: 957 - reason: Missing output files: output/data/runs/b30a-0209f019/202407170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407170000.ok - wildcards: run_id=b30a-0209f019, init_time=202407170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405230000.log - jobid: 792 - reason: Missing output files: output/data/runs/b30a-0209f019/202405230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405230000.ok - wildcards: run_id=b30a-0209f019, init_time=202405230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411070000.log - jobid: 1296 - reason: Missing output files: output/data/runs/b30a-0209f019/202411070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411070000.ok - wildcards: run_id=b30a-0209f019, init_time=202411070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407180000.log - jobid: 960 - reason: Missing output files: output/data/runs/b30a-0209f019/202407180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407180000.ok - wildcards: run_id=b30a-0209f019, init_time=202407180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409120000.log - jobid: 1128 - reason: Missing output files: output/data/runs/b30a-0209f019/202409120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409120000.ok - wildcards: run_id=b30a-0209f019, init_time=202409120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402010000.log - jobid: 456 - reason: Missing output files: output/data/runs/b30a-0209f019/202402010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402010000.ok - wildcards: run_id=b30a-0209f019, init_time=202402010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409130000.log - jobid: 1131 - reason: Missing output files: output/data/runs/b30a-0209f019/202409130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409130000.ok - wildcards: run_id=b30a-0209f019, init_time=202409130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402020000.log - jobid: 459 - reason: Missing output files: output/data/runs/b30a-0209f019/202402020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402020000.ok - wildcards: run_id=b30a-0209f019, init_time=202402020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411080000.log - jobid: 1299 - reason: Missing output files: output/data/runs/b30a-0209f019/202411080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411080000.ok - wildcards: run_id=b30a-0209f019, init_time=202411080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407190000.log - jobid: 963 - reason: Missing output files: output/data/runs/b30a-0209f019/202407190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407190000.ok - wildcards: run_id=b30a-0209f019, init_time=202407190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405240000.log - jobid: 795 - reason: Missing output files: output/data/runs/b30a-0209f019/202405240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405240000.ok - wildcards: run_id=b30a-0209f019, init_time=202405240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405250000.log - jobid: 798 - reason: Missing output files: output/data/runs/b30a-0209f019/202405250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405250000.ok - wildcards: run_id=b30a-0209f019, init_time=202405250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411090000.log - jobid: 1302 - reason: Missing output files: output/data/runs/b30a-0209f019/202411090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411090000.ok - wildcards: run_id=b30a-0209f019, init_time=202411090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407200000.log - jobid: 966 - reason: Missing output files: output/data/runs/b30a-0209f019/202407200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407200000.ok - wildcards: run_id=b30a-0209f019, init_time=202407200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402030000.log - jobid: 462 - reason: Missing output files: output/data/runs/b30a-0209f019/202402030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402030000.ok - wildcards: run_id=b30a-0209f019, init_time=202402030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409140000.log - jobid: 1134 - reason: Missing output files: output/data/runs/b30a-0209f019/202409140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409140000.ok - wildcards: run_id=b30a-0209f019, init_time=202409140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409150000.log - jobid: 1137 - reason: Missing output files: output/data/runs/b30a-0209f019/202409150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409150000.ok - wildcards: run_id=b30a-0209f019, init_time=202409150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402040000.log - jobid: 465 - reason: Missing output files: output/data/runs/b30a-0209f019/202402040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402040000.ok - wildcards: run_id=b30a-0209f019, init_time=202402040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407210000.log - jobid: 969 - reason: Missing output files: output/data/runs/b30a-0209f019/202407210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407210000.ok - wildcards: run_id=b30a-0209f019, init_time=202407210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411100000.log - jobid: 1305 - reason: Missing output files: output/data/runs/b30a-0209f019/202411100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411100000.ok - wildcards: run_id=b30a-0209f019, init_time=202411100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405260000.log - jobid: 801 - reason: Missing output files: output/data/runs/b30a-0209f019/202405260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405260000.ok - wildcards: run_id=b30a-0209f019, init_time=202405260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405270000.log - jobid: 804 - reason: Missing output files: output/data/runs/b30a-0209f019/202405270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405270000.ok - wildcards: run_id=b30a-0209f019, init_time=202405270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407220000.log - jobid: 972 - reason: Missing output files: output/data/runs/b30a-0209f019/202407220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407220000.ok - wildcards: run_id=b30a-0209f019, init_time=202407220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402050000.log - jobid: 468 - reason: Missing output files: output/data/runs/b30a-0209f019/202402050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402050000.ok - wildcards: run_id=b30a-0209f019, init_time=202402050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409160000.log - jobid: 1140 - reason: Missing output files: output/data/runs/b30a-0209f019/202409160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409160000.ok - wildcards: run_id=b30a-0209f019, init_time=202409160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411110000.log - jobid: 1308 - reason: Missing output files: output/data/runs/b30a-0209f019/202411110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411110000.ok - wildcards: run_id=b30a-0209f019, init_time=202411110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402060000.log - jobid: 471 - reason: Missing output files: output/data/runs/b30a-0209f019/202402060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402060000.ok - wildcards: run_id=b30a-0209f019, init_time=202402060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409170000.log - jobid: 1143 - reason: Missing output files: output/data/runs/b30a-0209f019/202409170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409170000.ok - wildcards: run_id=b30a-0209f019, init_time=202409170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407230000.log - jobid: 975 - reason: Missing output files: output/data/runs/b30a-0209f019/202407230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407230000.ok - wildcards: run_id=b30a-0209f019, init_time=202407230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411120000.log - jobid: 1311 - reason: Missing output files: output/data/runs/b30a-0209f019/202411120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411120000.ok - wildcards: run_id=b30a-0209f019, init_time=202411120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405280000.log - jobid: 807 - reason: Missing output files: output/data/runs/b30a-0209f019/202405280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405280000.ok - wildcards: run_id=b30a-0209f019, init_time=202405280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405290000.log - jobid: 810 - reason: Missing output files: output/data/runs/b30a-0209f019/202405290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405290000.ok - wildcards: run_id=b30a-0209f019, init_time=202405290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411130000.log - jobid: 1314 - reason: Missing output files: output/data/runs/b30a-0209f019/202411130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411130000.ok - wildcards: run_id=b30a-0209f019, init_time=202411130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407240000.log - jobid: 978 - reason: Missing output files: output/data/runs/b30a-0209f019/202407240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407240000.ok - wildcards: run_id=b30a-0209f019, init_time=202407240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409180000.log - jobid: 1146 - reason: Missing output files: output/data/runs/b30a-0209f019/202409180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409180000.ok - wildcards: run_id=b30a-0209f019, init_time=202409180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402070000.log - jobid: 474 - reason: Missing output files: output/data/runs/b30a-0209f019/202402070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402070000.ok - wildcards: run_id=b30a-0209f019, init_time=202402070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402080000.log - jobid: 477 - reason: Missing output files: output/data/runs/b30a-0209f019/202402080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402080000.ok - wildcards: run_id=b30a-0209f019, init_time=202402080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409190000.log - jobid: 1149 - reason: Missing output files: output/data/runs/b30a-0209f019/202409190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409190000.ok - wildcards: run_id=b30a-0209f019, init_time=202409190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407250000.log - jobid: 981 - reason: Missing output files: output/data/runs/b30a-0209f019/202407250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407250000.ok - wildcards: run_id=b30a-0209f019, init_time=202407250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411140000.log - jobid: 1317 - reason: Missing output files: output/data/runs/b30a-0209f019/202411140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411140000.ok - wildcards: run_id=b30a-0209f019, init_time=202411140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405300000.log - jobid: 813 - reason: Missing output files: output/data/runs/b30a-0209f019/202405300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405300000.ok - wildcards: run_id=b30a-0209f019, init_time=202405300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202405310000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202405310000.log - jobid: 816 - reason: Missing output files: output/data/runs/b30a-0209f019/202405310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405310000.ok - wildcards: run_id=b30a-0209f019, init_time=202405310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409200000.log - jobid: 1152 - reason: Missing output files: output/data/runs/b30a-0209f019/202409200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409200000.ok - wildcards: run_id=b30a-0209f019, init_time=202409200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402090000.log - jobid: 480 - reason: Missing output files: output/data/runs/b30a-0209f019/202402090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402090000.ok - wildcards: run_id=b30a-0209f019, init_time=202402090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407260000.log - jobid: 984 - reason: Missing output files: output/data/runs/b30a-0209f019/202407260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407260000.ok - wildcards: run_id=b30a-0209f019, init_time=202407260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411150000.log - jobid: 1320 - reason: Missing output files: output/data/runs/b30a-0209f019/202411150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411150000.ok - wildcards: run_id=b30a-0209f019, init_time=202411150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402100000.log - jobid: 483 - reason: Missing output files: output/data/runs/b30a-0209f019/202402100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402100000.ok - wildcards: run_id=b30a-0209f019, init_time=202402100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409210000.log - jobid: 1155 - reason: Missing output files: output/data/runs/b30a-0209f019/202409210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409210000.ok - wildcards: run_id=b30a-0209f019, init_time=202409210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407270000.log - jobid: 987 - reason: Missing output files: output/data/runs/b30a-0209f019/202407270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407270000.ok - wildcards: run_id=b30a-0209f019, init_time=202407270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411160000.log - jobid: 1323 - reason: Missing output files: output/data/runs/b30a-0209f019/202411160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411160000.ok - wildcards: run_id=b30a-0209f019, init_time=202411160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406010000.log - jobid: 819 - reason: Missing output files: output/data/runs/b30a-0209f019/202406010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406010000.ok - wildcards: run_id=b30a-0209f019, init_time=202406010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406020000.log - jobid: 822 - reason: Missing output files: output/data/runs/b30a-0209f019/202406020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406020000.ok - wildcards: run_id=b30a-0209f019, init_time=202406020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411170000.log - jobid: 1326 - reason: Missing output files: output/data/runs/b30a-0209f019/202411170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411170000.ok - wildcards: run_id=b30a-0209f019, init_time=202411170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407280000.log - jobid: 990 - reason: Missing output files: output/data/runs/b30a-0209f019/202407280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407280000.ok - wildcards: run_id=b30a-0209f019, init_time=202407280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409220000.log - jobid: 1158 - reason: Missing output files: output/data/runs/b30a-0209f019/202409220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409220000.ok - wildcards: run_id=b30a-0209f019, init_time=202409220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402110000.log - jobid: 486 - reason: Missing output files: output/data/runs/b30a-0209f019/202402110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402110000.ok - wildcards: run_id=b30a-0209f019, init_time=202402110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402120000.log - jobid: 489 - reason: Missing output files: output/data/runs/b30a-0209f019/202402120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402120000.ok - wildcards: run_id=b30a-0209f019, init_time=202402120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409230000.log - jobid: 1161 - reason: Missing output files: output/data/runs/b30a-0209f019/202409230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409230000.ok - wildcards: run_id=b30a-0209f019, init_time=202409230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407290000.log - jobid: 993 - reason: Missing output files: output/data/runs/b30a-0209f019/202407290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407290000.ok - wildcards: run_id=b30a-0209f019, init_time=202407290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411180000.log - jobid: 1329 - reason: Missing output files: output/data/runs/b30a-0209f019/202411180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411180000.ok - wildcards: run_id=b30a-0209f019, init_time=202411180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406030000.log - jobid: 825 - reason: Missing output files: output/data/runs/b30a-0209f019/202406030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406030000.ok - wildcards: run_id=b30a-0209f019, init_time=202406030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406040000.log - jobid: 828 - reason: Missing output files: output/data/runs/b30a-0209f019/202406040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406040000.ok - wildcards: run_id=b30a-0209f019, init_time=202406040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407300000.log - jobid: 996 - reason: Missing output files: output/data/runs/b30a-0209f019/202407300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407300000.ok - wildcards: run_id=b30a-0209f019, init_time=202407300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402130000.log - jobid: 492 - reason: Missing output files: output/data/runs/b30a-0209f019/202402130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402130000.ok - wildcards: run_id=b30a-0209f019, init_time=202402130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409240000.log - jobid: 1164 - reason: Missing output files: output/data/runs/b30a-0209f019/202409240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409240000.ok - wildcards: run_id=b30a-0209f019, init_time=202409240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411190000.log - jobid: 1332 - reason: Missing output files: output/data/runs/b30a-0209f019/202411190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411190000.ok - wildcards: run_id=b30a-0209f019, init_time=202411190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411200000.log - jobid: 1335 - reason: Missing output files: output/data/runs/b30a-0209f019/202411200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411200000.ok - wildcards: run_id=b30a-0209f019, init_time=202411200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402140000.log - jobid: 495 - reason: Missing output files: output/data/runs/b30a-0209f019/202402140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402140000.ok - wildcards: run_id=b30a-0209f019, init_time=202402140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202407310000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202407310000.log - jobid: 999 - reason: Missing output files: output/data/runs/b30a-0209f019/202407310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407310000.ok - wildcards: run_id=b30a-0209f019, init_time=202407310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409250000.log - jobid: 1167 - reason: Missing output files: output/data/runs/b30a-0209f019/202409250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409250000.ok - wildcards: run_id=b30a-0209f019, init_time=202409250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406050000.log - jobid: 831 - reason: Missing output files: output/data/runs/b30a-0209f019/202406050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406050000.ok - wildcards: run_id=b30a-0209f019, init_time=202406050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406060000.log - jobid: 834 - reason: Missing output files: output/data/runs/b30a-0209f019/202406060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406060000.ok - wildcards: run_id=b30a-0209f019, init_time=202406060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408010000.log - jobid: 1002 - reason: Missing output files: output/data/runs/b30a-0209f019/202408010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408010000.ok - wildcards: run_id=b30a-0209f019, init_time=202408010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402150000.log - jobid: 498 - reason: Missing output files: output/data/runs/b30a-0209f019/202402150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402150000.ok - wildcards: run_id=b30a-0209f019, init_time=202402150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411210000.log - jobid: 1338 - reason: Missing output files: output/data/runs/b30a-0209f019/202411210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411210000.ok - wildcards: run_id=b30a-0209f019, init_time=202411210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409260000.log - jobid: 1170 - reason: Missing output files: output/data/runs/b30a-0209f019/202409260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409260000.ok - wildcards: run_id=b30a-0209f019, init_time=202409260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408020000.log - jobid: 1005 - reason: Missing output files: output/data/runs/b30a-0209f019/202408020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408020000.ok - wildcards: run_id=b30a-0209f019, init_time=202408020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402160000.log - jobid: 501 - reason: Missing output files: output/data/runs/b30a-0209f019/202402160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402160000.ok - wildcards: run_id=b30a-0209f019, init_time=202402160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409270000.log - jobid: 1173 - reason: Missing output files: output/data/runs/b30a-0209f019/202409270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409270000.ok - wildcards: run_id=b30a-0209f019, init_time=202409270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411220000.log - jobid: 1341 - reason: Missing output files: output/data/runs/b30a-0209f019/202411220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411220000.ok - wildcards: run_id=b30a-0209f019, init_time=202411220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406070000.log - jobid: 837 - reason: Missing output files: output/data/runs/b30a-0209f019/202406070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406070000.ok - wildcards: run_id=b30a-0209f019, init_time=202406070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406080000.log - jobid: 840 - reason: Missing output files: output/data/runs/b30a-0209f019/202406080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406080000.ok - wildcards: run_id=b30a-0209f019, init_time=202406080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411230000.log - jobid: 1344 - reason: Missing output files: output/data/runs/b30a-0209f019/202411230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411230000.ok - wildcards: run_id=b30a-0209f019, init_time=202411230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409280000.log - jobid: 1176 - reason: Missing output files: output/data/runs/b30a-0209f019/202409280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409280000.ok - wildcards: run_id=b30a-0209f019, init_time=202409280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408030000.log - jobid: 1008 - reason: Missing output files: output/data/runs/b30a-0209f019/202408030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408030000.ok - wildcards: run_id=b30a-0209f019, init_time=202408030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402170000.log - jobid: 504 - reason: Missing output files: output/data/runs/b30a-0209f019/202402170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402170000.ok - wildcards: run_id=b30a-0209f019, init_time=202402170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402180000.log - jobid: 507 - reason: Missing output files: output/data/runs/b30a-0209f019/202402180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402180000.ok - wildcards: run_id=b30a-0209f019, init_time=202402180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408040000.log - jobid: 1011 - reason: Missing output files: output/data/runs/b30a-0209f019/202408040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408040000.ok - wildcards: run_id=b30a-0209f019, init_time=202408040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409290000.log - jobid: 1179 - reason: Missing output files: output/data/runs/b30a-0209f019/202409290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409290000.ok - wildcards: run_id=b30a-0209f019, init_time=202409290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411240000.log - jobid: 1347 - reason: Missing output files: output/data/runs/b30a-0209f019/202411240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411240000.ok - wildcards: run_id=b30a-0209f019, init_time=202411240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406090000.log - jobid: 843 - reason: Missing output files: output/data/runs/b30a-0209f019/202406090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406090000.ok - wildcards: run_id=b30a-0209f019, init_time=202406090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406100000.log - jobid: 846 - reason: Missing output files: output/data/runs/b30a-0209f019/202406100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406100000.ok - wildcards: run_id=b30a-0209f019, init_time=202406100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411250000.log - jobid: 1350 - reason: Missing output files: output/data/runs/b30a-0209f019/202411250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411250000.ok - wildcards: run_id=b30a-0209f019, init_time=202411250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202409300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202409300000.log - jobid: 1182 - reason: Missing output files: output/data/runs/b30a-0209f019/202409300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409300000.ok - wildcards: run_id=b30a-0209f019, init_time=202409300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408050000.log - jobid: 1014 - reason: Missing output files: output/data/runs/b30a-0209f019/202408050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408050000.ok - wildcards: run_id=b30a-0209f019, init_time=202408050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402190000.log - jobid: 510 - reason: Missing output files: output/data/runs/b30a-0209f019/202402190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402190000.ok - wildcards: run_id=b30a-0209f019, init_time=202402190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402200000.log - jobid: 513 - reason: Missing output files: output/data/runs/b30a-0209f019/202402200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402200000.ok - wildcards: run_id=b30a-0209f019, init_time=202402200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408060000.log - jobid: 1017 - reason: Missing output files: output/data/runs/b30a-0209f019/202408060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408060000.ok - wildcards: run_id=b30a-0209f019, init_time=202408060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410010000.log - jobid: 1185 - reason: Missing output files: output/data/runs/b30a-0209f019/202410010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410010000.ok - wildcards: run_id=b30a-0209f019, init_time=202410010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411260000.log - jobid: 1353 - reason: Missing output files: output/data/runs/b30a-0209f019/202411260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411260000.ok - wildcards: run_id=b30a-0209f019, init_time=202411260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406110000.log - jobid: 849 - reason: Missing output files: output/data/runs/b30a-0209f019/202406110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406110000.ok - wildcards: run_id=b30a-0209f019, init_time=202406110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406120000.log - jobid: 852 - reason: Missing output files: output/data/runs/b30a-0209f019/202406120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406120000.ok - wildcards: run_id=b30a-0209f019, init_time=202406120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411270000.log - jobid: 1356 - reason: Missing output files: output/data/runs/b30a-0209f019/202411270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411270000.ok - wildcards: run_id=b30a-0209f019, init_time=202411270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410020000.log - jobid: 1188 - reason: Missing output files: output/data/runs/b30a-0209f019/202410020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410020000.ok - wildcards: run_id=b30a-0209f019, init_time=202410020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408070000.log - jobid: 1020 - reason: Missing output files: output/data/runs/b30a-0209f019/202408070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408070000.ok - wildcards: run_id=b30a-0209f019, init_time=202408070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402210000.log - jobid: 516 - reason: Missing output files: output/data/runs/b30a-0209f019/202402210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402210000.ok - wildcards: run_id=b30a-0209f019, init_time=202402210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402220000.log - jobid: 519 - reason: Missing output files: output/data/runs/b30a-0209f019/202402220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402220000.ok - wildcards: run_id=b30a-0209f019, init_time=202402220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408080000.log - jobid: 1023 - reason: Missing output files: output/data/runs/b30a-0209f019/202408080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408080000.ok - wildcards: run_id=b30a-0209f019, init_time=202408080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410030000.log - jobid: 1191 - reason: Missing output files: output/data/runs/b30a-0209f019/202410030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410030000.ok - wildcards: run_id=b30a-0209f019, init_time=202410030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406130000.log - jobid: 855 - reason: Missing output files: output/data/runs/b30a-0209f019/202406130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406130000.ok - wildcards: run_id=b30a-0209f019, init_time=202406130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411280000.log - jobid: 1359 - reason: Missing output files: output/data/runs/b30a-0209f019/202411280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411280000.ok - wildcards: run_id=b30a-0209f019, init_time=202411280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406140000.log - jobid: 858 - reason: Missing output files: output/data/runs/b30a-0209f019/202406140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406140000.ok - wildcards: run_id=b30a-0209f019, init_time=202406140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411290000.log - jobid: 1362 - reason: Missing output files: output/data/runs/b30a-0209f019/202411290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411290000.ok - wildcards: run_id=b30a-0209f019, init_time=202411290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408090000.log - jobid: 1026 - reason: Missing output files: output/data/runs/b30a-0209f019/202408090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408090000.ok - wildcards: run_id=b30a-0209f019, init_time=202408090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410040000.log - jobid: 1194 - reason: Missing output files: output/data/runs/b30a-0209f019/202410040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410040000.ok - wildcards: run_id=b30a-0209f019, init_time=202410040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402230000.log - jobid: 522 - reason: Missing output files: output/data/runs/b30a-0209f019/202402230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402230000.ok - wildcards: run_id=b30a-0209f019, init_time=202402230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408100000.log - jobid: 1029 - reason: Missing output files: output/data/runs/b30a-0209f019/202408100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408100000.ok - wildcards: run_id=b30a-0209f019, init_time=202408100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402240000.log - jobid: 525 - reason: Missing output files: output/data/runs/b30a-0209f019/202402240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402240000.ok - wildcards: run_id=b30a-0209f019, init_time=202402240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410050000.log - jobid: 1197 - reason: Missing output files: output/data/runs/b30a-0209f019/202410050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410050000.ok - wildcards: run_id=b30a-0209f019, init_time=202410050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406150000.log - jobid: 861 - reason: Missing output files: output/data/runs/b30a-0209f019/202406150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406150000.ok - wildcards: run_id=b30a-0209f019, init_time=202406150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202411300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202411300000.log - jobid: 1365 - reason: Missing output files: output/data/runs/b30a-0209f019/202411300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411300000.ok - wildcards: run_id=b30a-0209f019, init_time=202411300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412010000.log - jobid: 1368 - reason: Missing output files: output/data/runs/b30a-0209f019/202412010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412010000.ok - wildcards: run_id=b30a-0209f019, init_time=202412010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406160000.log - jobid: 864 - reason: Missing output files: output/data/runs/b30a-0209f019/202406160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406160000.ok - wildcards: run_id=b30a-0209f019, init_time=202406160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410060000.log - jobid: 1200 - reason: Missing output files: output/data/runs/b30a-0209f019/202410060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410060000.ok - wildcards: run_id=b30a-0209f019, init_time=202410060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408110000.log - jobid: 1032 - reason: Missing output files: output/data/runs/b30a-0209f019/202408110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408110000.ok - wildcards: run_id=b30a-0209f019, init_time=202408110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402250000.log - jobid: 528 - reason: Missing output files: output/data/runs/b30a-0209f019/202402250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402250000.ok - wildcards: run_id=b30a-0209f019, init_time=202402250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408120000.log - jobid: 1035 - reason: Missing output files: output/data/runs/b30a-0209f019/202408120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408120000.ok - wildcards: run_id=b30a-0209f019, init_time=202408120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402260000.log - jobid: 531 - reason: Missing output files: output/data/runs/b30a-0209f019/202402260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402260000.ok - wildcards: run_id=b30a-0209f019, init_time=202402260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410070000.log - jobid: 1203 - reason: Missing output files: output/data/runs/b30a-0209f019/202410070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410070000.ok - wildcards: run_id=b30a-0209f019, init_time=202410070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412020000.log - jobid: 1371 - reason: Missing output files: output/data/runs/b30a-0209f019/202412020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412020000.ok - wildcards: run_id=b30a-0209f019, init_time=202412020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406170000.log - jobid: 867 - reason: Missing output files: output/data/runs/b30a-0209f019/202406170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406170000.ok - wildcards: run_id=b30a-0209f019, init_time=202406170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202412030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202412030000.log - jobid: 1374 - reason: Missing output files: output/data/runs/b30a-0209f019/202412030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412030000.ok - wildcards: run_id=b30a-0209f019, init_time=202412030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202406180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202406180000.log - jobid: 870 - reason: Missing output files: output/data/runs/b30a-0209f019/202406180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406180000.ok - wildcards: run_id=b30a-0209f019, init_time=202406180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202410080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202410080000.log - jobid: 1206 - reason: Missing output files: output/data/runs/b30a-0209f019/202410080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410080000.ok - wildcards: run_id=b30a-0209f019, init_time=202410080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402270000.log - jobid: 534 - reason: Missing output files: output/data/runs/b30a-0209f019/202402270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402270000.ok - wildcards: run_id=b30a-0209f019, init_time=202402270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202408130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202408130000.log - jobid: 1038 - reason: Missing output files: output/data/runs/b30a-0209f019/202408130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408130000.ok - wildcards: run_id=b30a-0209f019, init_time=202408130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402280000.log - jobid: 537 - reason: Missing output files: output/data/runs/b30a-0209f019/202402280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402280000.ok - wildcards: run_id=b30a-0209f019, init_time=202402280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202402290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202402290000.log - jobid: 540 - reason: Missing output files: output/data/runs/b30a-0209f019/202402290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402290000.ok - wildcards: run_id=b30a-0209f019, init_time=202402290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403010000.log - jobid: 543 - reason: Missing output files: output/data/runs/b30a-0209f019/202403010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403010000.ok - wildcards: run_id=b30a-0209f019, init_time=202403010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403020000.log - jobid: 546 - reason: Missing output files: output/data/runs/b30a-0209f019/202403020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403020000.ok - wildcards: run_id=b30a-0209f019, init_time=202403020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403030000.log - jobid: 549 - reason: Missing output files: output/data/runs/b30a-0209f019/202403030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403030000.ok - wildcards: run_id=b30a-0209f019, init_time=202403030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403040000.log - jobid: 552 - reason: Missing output files: output/data/runs/b30a-0209f019/202403040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403040000.ok - wildcards: run_id=b30a-0209f019, init_time=202403040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403050000.log - jobid: 555 - reason: Missing output files: output/data/runs/b30a-0209f019/202403050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403050000.ok - wildcards: run_id=b30a-0209f019, init_time=202403050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403060000.log - jobid: 558 - reason: Missing output files: output/data/runs/b30a-0209f019/202403060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403060000.ok - wildcards: run_id=b30a-0209f019, init_time=202403060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403070000.log - jobid: 561 - reason: Missing output files: output/data/runs/b30a-0209f019/202403070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403070000.ok - wildcards: run_id=b30a-0209f019, init_time=202403070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403080000.log - jobid: 564 - reason: Missing output files: output/data/runs/b30a-0209f019/202403080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403080000.ok - wildcards: run_id=b30a-0209f019, init_time=202403080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403090000.log - jobid: 567 - reason: Missing output files: output/data/runs/b30a-0209f019/202403090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403090000.ok - wildcards: run_id=b30a-0209f019, init_time=202403090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403100000.log - jobid: 570 - reason: Missing output files: output/data/runs/b30a-0209f019/202403100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403100000.ok - wildcards: run_id=b30a-0209f019, init_time=202403100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403110000.log - jobid: 573 - reason: Missing output files: output/data/runs/b30a-0209f019/202403110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403110000.ok - wildcards: run_id=b30a-0209f019, init_time=202403110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403120000.log - jobid: 576 - reason: Missing output files: output/data/runs/b30a-0209f019/202403120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403120000.ok - wildcards: run_id=b30a-0209f019, init_time=202403120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403130000.log - jobid: 579 - reason: Missing output files: output/data/runs/b30a-0209f019/202403130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403130000.ok - wildcards: run_id=b30a-0209f019, init_time=202403130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403140000.log - jobid: 582 - reason: Missing output files: output/data/runs/b30a-0209f019/202403140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403140000.ok - wildcards: run_id=b30a-0209f019, init_time=202403140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403150000.log - jobid: 585 - reason: Missing output files: output/data/runs/b30a-0209f019/202403150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403150000.ok - wildcards: run_id=b30a-0209f019, init_time=202403150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403160000.log - jobid: 588 - reason: Missing output files: output/data/runs/b30a-0209f019/202403160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403160000.ok - wildcards: run_id=b30a-0209f019, init_time=202403160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403170000.log - jobid: 591 - reason: Missing output files: output/data/runs/b30a-0209f019/202403170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403170000.ok - wildcards: run_id=b30a-0209f019, init_time=202403170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403180000.log - jobid: 594 - reason: Missing output files: output/data/runs/b30a-0209f019/202403180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403180000.ok - wildcards: run_id=b30a-0209f019, init_time=202403180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403190000.log - jobid: 597 - reason: Missing output files: output/data/runs/b30a-0209f019/202403190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403190000.ok - wildcards: run_id=b30a-0209f019, init_time=202403190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403200000.log - jobid: 600 - reason: Missing output files: output/data/runs/b30a-0209f019/202403200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403200000.ok - wildcards: run_id=b30a-0209f019, init_time=202403200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403210000.log - jobid: 603 - reason: Missing output files: output/data/runs/b30a-0209f019/202403210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403210000.ok - wildcards: run_id=b30a-0209f019, init_time=202403210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403220000.log - jobid: 606 - reason: Missing output files: output/data/runs/b30a-0209f019/202403220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403220000.ok - wildcards: run_id=b30a-0209f019, init_time=202403220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403230000.log - jobid: 609 - reason: Missing output files: output/data/runs/b30a-0209f019/202403230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403230000.ok - wildcards: run_id=b30a-0209f019, init_time=202403230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403240000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403240000.log - jobid: 612 - reason: Missing output files: output/data/runs/b30a-0209f019/202403240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403240000.ok - wildcards: run_id=b30a-0209f019, init_time=202403240000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403250000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403250000.log - jobid: 615 - reason: Missing output files: output/data/runs/b30a-0209f019/202403250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403250000.ok - wildcards: run_id=b30a-0209f019, init_time=202403250000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403260000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403260000.log - jobid: 618 - reason: Missing output files: output/data/runs/b30a-0209f019/202403260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403260000.ok - wildcards: run_id=b30a-0209f019, init_time=202403260000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403270000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403270000.log - jobid: 621 - reason: Missing output files: output/data/runs/b30a-0209f019/202403270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403270000.ok - wildcards: run_id=b30a-0209f019, init_time=202403270000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403280000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403280000.log - jobid: 624 - reason: Missing output files: output/data/runs/b30a-0209f019/202403280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403280000.ok - wildcards: run_id=b30a-0209f019, init_time=202403280000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403290000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403290000.log - jobid: 627 - reason: Missing output files: output/data/runs/b30a-0209f019/202403290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403290000.ok - wildcards: run_id=b30a-0209f019, init_time=202403290000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403300000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403300000.log - jobid: 630 - reason: Missing output files: output/data/runs/b30a-0209f019/202403300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403300000.ok - wildcards: run_id=b30a-0209f019, init_time=202403300000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202403310000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202403310000.log - jobid: 633 - reason: Missing output files: output/data/runs/b30a-0209f019/202403310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403310000.ok - wildcards: run_id=b30a-0209f019, init_time=202403310000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404010000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404010000.log - jobid: 636 - reason: Missing output files: output/data/runs/b30a-0209f019/202404010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404010000.ok - wildcards: run_id=b30a-0209f019, init_time=202404010000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404020000.log - jobid: 639 - reason: Missing output files: output/data/runs/b30a-0209f019/202404020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404020000.ok - wildcards: run_id=b30a-0209f019, init_time=202404020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404030000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404030000.log - jobid: 642 - reason: Missing output files: output/data/runs/b30a-0209f019/202404030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404030000.ok - wildcards: run_id=b30a-0209f019, init_time=202404030000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404040000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404040000.log - jobid: 645 - reason: Missing output files: output/data/runs/b30a-0209f019/202404040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404040000.ok - wildcards: run_id=b30a-0209f019, init_time=202404040000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404050000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404050000.log - jobid: 648 - reason: Missing output files: output/data/runs/b30a-0209f019/202404050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404050000.ok - wildcards: run_id=b30a-0209f019, init_time=202404050000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404060000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404060000.log - jobid: 651 - reason: Missing output files: output/data/runs/b30a-0209f019/202404060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404060000.ok - wildcards: run_id=b30a-0209f019, init_time=202404060000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404070000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404070000.log - jobid: 654 - reason: Missing output files: output/data/runs/b30a-0209f019/202404070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404070000.ok - wildcards: run_id=b30a-0209f019, init_time=202404070000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404080000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404080000.log - jobid: 657 - reason: Missing output files: output/data/runs/b30a-0209f019/202404080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404080000.ok - wildcards: run_id=b30a-0209f019, init_time=202404080000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404090000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404090000.log - jobid: 660 - reason: Missing output files: output/data/runs/b30a-0209f019/202404090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404090000.ok - wildcards: run_id=b30a-0209f019, init_time=202404090000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404100000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404100000.log - jobid: 663 - reason: Missing output files: output/data/runs/b30a-0209f019/202404100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404100000.ok - wildcards: run_id=b30a-0209f019, init_time=202404100000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404110000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404110000.log - jobid: 666 - reason: Missing output files: output/data/runs/b30a-0209f019/202404110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404110000.ok - wildcards: run_id=b30a-0209f019, init_time=202404110000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404120000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404120000.log - jobid: 669 - reason: Missing output files: output/data/runs/b30a-0209f019/202404120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404120000.ok - wildcards: run_id=b30a-0209f019, init_time=202404120000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404130000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404130000.log - jobid: 672 - reason: Missing output files: output/data/runs/b30a-0209f019/202404130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404130000.ok - wildcards: run_id=b30a-0209f019, init_time=202404130000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404140000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404140000.log - jobid: 675 - reason: Missing output files: output/data/runs/b30a-0209f019/202404140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404140000.ok - wildcards: run_id=b30a-0209f019, init_time=202404140000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404150000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404150000.log - jobid: 678 - reason: Missing output files: output/data/runs/b30a-0209f019/202404150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404150000.ok - wildcards: run_id=b30a-0209f019, init_time=202404150000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404160000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404160000.log - jobid: 681 - reason: Missing output files: output/data/runs/b30a-0209f019/202404160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404160000.ok - wildcards: run_id=b30a-0209f019, init_time=202404160000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404170000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404170000.log - jobid: 684 - reason: Missing output files: output/data/runs/b30a-0209f019/202404170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404170000.ok - wildcards: run_id=b30a-0209f019, init_time=202404170000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404180000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404180000.log - jobid: 687 - reason: Missing output files: output/data/runs/b30a-0209f019/202404180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404180000.ok - wildcards: run_id=b30a-0209f019, init_time=202404180000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404190000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404190000.log - jobid: 690 - reason: Missing output files: output/data/runs/b30a-0209f019/202404190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404190000.ok - wildcards: run_id=b30a-0209f019, init_time=202404190000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404200000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404200000.log - jobid: 693 - reason: Missing output files: output/data/runs/b30a-0209f019/202404200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404200000.ok - wildcards: run_id=b30a-0209f019, init_time=202404200000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404210000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404210000.log - jobid: 696 - reason: Missing output files: output/data/runs/b30a-0209f019/202404210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404210000.ok - wildcards: run_id=b30a-0209f019, init_time=202404210000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404220000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404220000.log - jobid: 699 - reason: Missing output files: output/data/runs/b30a-0209f019/202404220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404220000.ok - wildcards: run_id=b30a-0209f019, init_time=202404220000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202401020000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202401020000.log - jobid: 363 - reason: Missing output files: output/data/runs/b30a-0209f019/202401020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401020000.ok - wildcards: run_id=b30a-0209f019, init_time=202401020000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics: - input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - output: output/data/runs/b30a-0209f019/202404230000/verif.nc - log: output/logs/verif_metrics/b30a-0209f019-202404230000.log - jobid: 702 - reason: Missing output files: output/data/runs/b30a-0209f019/202404230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404230000.ok - wildcards: run_id=b30a-0209f019, init_time=202404230000 - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_aggregation: - input: workflow/scripts/verif_aggregation.py, output/data/runs/b30a-0209f019/202401020000/verif.nc, output/data/runs/b30a-0209f019/202401030000/verif.nc, output/data/runs/b30a-0209f019/202401040000/verif.nc, output/data/runs/b30a-0209f019/202401050000/verif.nc, output/data/runs/b30a-0209f019/202401060000/verif.nc, output/data/runs/b30a-0209f019/202401070000/verif.nc, output/data/runs/b30a-0209f019/202401080000/verif.nc, output/data/runs/b30a-0209f019/202401090000/verif.nc, output/data/runs/b30a-0209f019/202401100000/verif.nc, output/data/runs/b30a-0209f019/202401110000/verif.nc, output/data/runs/b30a-0209f019/202401120000/verif.nc, output/data/runs/b30a-0209f019/202401130000/verif.nc, output/data/runs/b30a-0209f019/202401140000/verif.nc, output/data/runs/b30a-0209f019/202401150000/verif.nc, output/data/runs/b30a-0209f019/202401160000/verif.nc, output/data/runs/b30a-0209f019/202401170000/verif.nc, output/data/runs/b30a-0209f019/202401180000/verif.nc, output/data/runs/b30a-0209f019/202401190000/verif.nc, output/data/runs/b30a-0209f019/202401200000/verif.nc, output/data/runs/b30a-0209f019/202401210000/verif.nc, output/data/runs/b30a-0209f019/202401220000/verif.nc, output/data/runs/b30a-0209f019/202401230000/verif.nc, output/data/runs/b30a-0209f019/202401240000/verif.nc, output/data/runs/b30a-0209f019/202401250000/verif.nc, output/data/runs/b30a-0209f019/202401260000/verif.nc, output/data/runs/b30a-0209f019/202401270000/verif.nc, output/data/runs/b30a-0209f019/202401280000/verif.nc, output/data/runs/b30a-0209f019/202401290000/verif.nc, output/data/runs/b30a-0209f019/202401300000/verif.nc, output/data/runs/b30a-0209f019/202401310000/verif.nc, output/data/runs/b30a-0209f019/202402010000/verif.nc, output/data/runs/b30a-0209f019/202402020000/verif.nc, output/data/runs/b30a-0209f019/202402030000/verif.nc, output/data/runs/b30a-0209f019/202402040000/verif.nc, output/data/runs/b30a-0209f019/202402050000/verif.nc, output/data/runs/b30a-0209f019/202402060000/verif.nc, output/data/runs/b30a-0209f019/202402070000/verif.nc, output/data/runs/b30a-0209f019/202402080000/verif.nc, output/data/runs/b30a-0209f019/202402090000/verif.nc, output/data/runs/b30a-0209f019/202402100000/verif.nc, output/data/runs/b30a-0209f019/202402110000/verif.nc, output/data/runs/b30a-0209f019/202402120000/verif.nc, output/data/runs/b30a-0209f019/202402130000/verif.nc, output/data/runs/b30a-0209f019/202402140000/verif.nc, output/data/runs/b30a-0209f019/202402150000/verif.nc, output/data/runs/b30a-0209f019/202402160000/verif.nc, output/data/runs/b30a-0209f019/202402170000/verif.nc, output/data/runs/b30a-0209f019/202402180000/verif.nc, output/data/runs/b30a-0209f019/202402190000/verif.nc, output/data/runs/b30a-0209f019/202402200000/verif.nc, output/data/runs/b30a-0209f019/202402210000/verif.nc, output/data/runs/b30a-0209f019/202402220000/verif.nc, output/data/runs/b30a-0209f019/202402230000/verif.nc, output/data/runs/b30a-0209f019/202402240000/verif.nc, output/data/runs/b30a-0209f019/202402250000/verif.nc, output/data/runs/b30a-0209f019/202402260000/verif.nc, output/data/runs/b30a-0209f019/202402270000/verif.nc, output/data/runs/b30a-0209f019/202402280000/verif.nc, output/data/runs/b30a-0209f019/202402290000/verif.nc, output/data/runs/b30a-0209f019/202403010000/verif.nc, output/data/runs/b30a-0209f019/202403020000/verif.nc, output/data/runs/b30a-0209f019/202403030000/verif.nc, output/data/runs/b30a-0209f019/202403040000/verif.nc, output/data/runs/b30a-0209f019/202403050000/verif.nc, output/data/runs/b30a-0209f019/202403060000/verif.nc, output/data/runs/b30a-0209f019/202403070000/verif.nc, output/data/runs/b30a-0209f019/202403080000/verif.nc, output/data/runs/b30a-0209f019/202403090000/verif.nc, output/data/runs/b30a-0209f019/202403100000/verif.nc, output/data/runs/b30a-0209f019/202403110000/verif.nc, output/data/runs/b30a-0209f019/202403120000/verif.nc, output/data/runs/b30a-0209f019/202403130000/verif.nc, output/data/runs/b30a-0209f019/202403140000/verif.nc, output/data/runs/b30a-0209f019/202403150000/verif.nc, output/data/runs/b30a-0209f019/202403160000/verif.nc, output/data/runs/b30a-0209f019/202403170000/verif.nc, output/data/runs/b30a-0209f019/202403180000/verif.nc, output/data/runs/b30a-0209f019/202403190000/verif.nc, output/data/runs/b30a-0209f019/202403200000/verif.nc, output/data/runs/b30a-0209f019/202403210000/verif.nc, output/data/runs/b30a-0209f019/202403220000/verif.nc, output/data/runs/b30a-0209f019/202403230000/verif.nc, output/data/runs/b30a-0209f019/202403240000/verif.nc, output/data/runs/b30a-0209f019/202403250000/verif.nc, output/data/runs/b30a-0209f019/202403260000/verif.nc, output/data/runs/b30a-0209f019/202403270000/verif.nc, output/data/runs/b30a-0209f019/202403280000/verif.nc, output/data/runs/b30a-0209f019/202403290000/verif.nc, output/data/runs/b30a-0209f019/202403300000/verif.nc, output/data/runs/b30a-0209f019/202403310000/verif.nc, output/data/runs/b30a-0209f019/202404010000/verif.nc, output/data/runs/b30a-0209f019/202404020000/verif.nc, output/data/runs/b30a-0209f019/202404030000/verif.nc, output/data/runs/b30a-0209f019/202404040000/verif.nc, output/data/runs/b30a-0209f019/202404050000/verif.nc, output/data/runs/b30a-0209f019/202404060000/verif.nc, output/data/runs/b30a-0209f019/202404070000/verif.nc, output/data/runs/b30a-0209f019/202404080000/verif.nc, output/data/runs/b30a-0209f019/202404090000/verif.nc, output/data/runs/b30a-0209f019/202404100000/verif.nc, output/data/runs/b30a-0209f019/202404110000/verif.nc, output/data/runs/b30a-0209f019/202404120000/verif.nc, output/data/runs/b30a-0209f019/202404130000/verif.nc, output/data/runs/b30a-0209f019/202404140000/verif.nc, output/data/runs/b30a-0209f019/202404150000/verif.nc, output/data/runs/b30a-0209f019/202404160000/verif.nc, output/data/runs/b30a-0209f019/202404170000/verif.nc, output/data/runs/b30a-0209f019/202404180000/verif.nc, output/data/runs/b30a-0209f019/202404190000/verif.nc, output/data/runs/b30a-0209f019/202404200000/verif.nc, output/data/runs/b30a-0209f019/202404210000/verif.nc, output/data/runs/b30a-0209f019/202404220000/verif.nc, output/data/runs/b30a-0209f019/202404230000/verif.nc, output/data/runs/b30a-0209f019/202404240000/verif.nc, output/data/runs/b30a-0209f019/202404250000/verif.nc, output/data/runs/b30a-0209f019/202404260000/verif.nc, output/data/runs/b30a-0209f019/202404270000/verif.nc, output/data/runs/b30a-0209f019/202404280000/verif.nc, output/data/runs/b30a-0209f019/202404290000/verif.nc, output/data/runs/b30a-0209f019/202404300000/verif.nc, output/data/runs/b30a-0209f019/202405010000/verif.nc, output/data/runs/b30a-0209f019/202405020000/verif.nc, output/data/runs/b30a-0209f019/202405030000/verif.nc, output/data/runs/b30a-0209f019/202405040000/verif.nc, output/data/runs/b30a-0209f019/202405050000/verif.nc, output/data/runs/b30a-0209f019/202405060000/verif.nc, output/data/runs/b30a-0209f019/202405070000/verif.nc, output/data/runs/b30a-0209f019/202405080000/verif.nc, output/data/runs/b30a-0209f019/202405090000/verif.nc, output/data/runs/b30a-0209f019/202405100000/verif.nc, output/data/runs/b30a-0209f019/202405110000/verif.nc, output/data/runs/b30a-0209f019/202405120000/verif.nc, output/data/runs/b30a-0209f019/202405130000/verif.nc, output/data/runs/b30a-0209f019/202405140000/verif.nc, output/data/runs/b30a-0209f019/202405150000/verif.nc, output/data/runs/b30a-0209f019/202405160000/verif.nc, output/data/runs/b30a-0209f019/202405170000/verif.nc, output/data/runs/b30a-0209f019/202405180000/verif.nc, output/data/runs/b30a-0209f019/202405190000/verif.nc, output/data/runs/b30a-0209f019/202405200000/verif.nc, output/data/runs/b30a-0209f019/202405210000/verif.nc, output/data/runs/b30a-0209f019/202405220000/verif.nc, output/data/runs/b30a-0209f019/202405230000/verif.nc, output/data/runs/b30a-0209f019/202405240000/verif.nc, output/data/runs/b30a-0209f019/202405250000/verif.nc, output/data/runs/b30a-0209f019/202405260000/verif.nc, output/data/runs/b30a-0209f019/202405270000/verif.nc, output/data/runs/b30a-0209f019/202405280000/verif.nc, output/data/runs/b30a-0209f019/202405290000/verif.nc, output/data/runs/b30a-0209f019/202405300000/verif.nc, output/data/runs/b30a-0209f019/202405310000/verif.nc, output/data/runs/b30a-0209f019/202406010000/verif.nc, output/data/runs/b30a-0209f019/202406020000/verif.nc, output/data/runs/b30a-0209f019/202406030000/verif.nc, output/data/runs/b30a-0209f019/202406040000/verif.nc, output/data/runs/b30a-0209f019/202406050000/verif.nc, output/data/runs/b30a-0209f019/202406060000/verif.nc, output/data/runs/b30a-0209f019/202406070000/verif.nc, output/data/runs/b30a-0209f019/202406080000/verif.nc, output/data/runs/b30a-0209f019/202406090000/verif.nc, output/data/runs/b30a-0209f019/202406100000/verif.nc, output/data/runs/b30a-0209f019/202406110000/verif.nc, output/data/runs/b30a-0209f019/202406120000/verif.nc, output/data/runs/b30a-0209f019/202406130000/verif.nc, output/data/runs/b30a-0209f019/202406140000/verif.nc, output/data/runs/b30a-0209f019/202406150000/verif.nc, output/data/runs/b30a-0209f019/202406160000/verif.nc, output/data/runs/b30a-0209f019/202406170000/verif.nc, output/data/runs/b30a-0209f019/202406180000/verif.nc, output/data/runs/b30a-0209f019/202406190000/verif.nc, output/data/runs/b30a-0209f019/202406200000/verif.nc, output/data/runs/b30a-0209f019/202406210000/verif.nc, output/data/runs/b30a-0209f019/202406220000/verif.nc, output/data/runs/b30a-0209f019/202406230000/verif.nc, output/data/runs/b30a-0209f019/202406240000/verif.nc, output/data/runs/b30a-0209f019/202406250000/verif.nc, output/data/runs/b30a-0209f019/202406260000/verif.nc, output/data/runs/b30a-0209f019/202406270000/verif.nc, output/data/runs/b30a-0209f019/202406280000/verif.nc, output/data/runs/b30a-0209f019/202406290000/verif.nc, output/data/runs/b30a-0209f019/202406300000/verif.nc, output/data/runs/b30a-0209f019/202407010000/verif.nc, output/data/runs/b30a-0209f019/202407020000/verif.nc, output/data/runs/b30a-0209f019/202407030000/verif.nc, output/data/runs/b30a-0209f019/202407040000/verif.nc, output/data/runs/b30a-0209f019/202407050000/verif.nc, output/data/runs/b30a-0209f019/202407060000/verif.nc, output/data/runs/b30a-0209f019/202407070000/verif.nc, output/data/runs/b30a-0209f019/202407080000/verif.nc, output/data/runs/b30a-0209f019/202407090000/verif.nc, output/data/runs/b30a-0209f019/202407100000/verif.nc, output/data/runs/b30a-0209f019/202407110000/verif.nc, output/data/runs/b30a-0209f019/202407120000/verif.nc, output/data/runs/b30a-0209f019/202407130000/verif.nc, output/data/runs/b30a-0209f019/202407140000/verif.nc, output/data/runs/b30a-0209f019/202407150000/verif.nc, output/data/runs/b30a-0209f019/202407160000/verif.nc, output/data/runs/b30a-0209f019/202407170000/verif.nc, output/data/runs/b30a-0209f019/202407180000/verif.nc, output/data/runs/b30a-0209f019/202407190000/verif.nc, output/data/runs/b30a-0209f019/202407200000/verif.nc, output/data/runs/b30a-0209f019/202407210000/verif.nc, output/data/runs/b30a-0209f019/202407220000/verif.nc, output/data/runs/b30a-0209f019/202407230000/verif.nc, output/data/runs/b30a-0209f019/202407240000/verif.nc, output/data/runs/b30a-0209f019/202407250000/verif.nc, output/data/runs/b30a-0209f019/202407260000/verif.nc, output/data/runs/b30a-0209f019/202407270000/verif.nc, output/data/runs/b30a-0209f019/202407280000/verif.nc, output/data/runs/b30a-0209f019/202407290000/verif.nc, output/data/runs/b30a-0209f019/202407300000/verif.nc, output/data/runs/b30a-0209f019/202407310000/verif.nc, output/data/runs/b30a-0209f019/202408010000/verif.nc, output/data/runs/b30a-0209f019/202408020000/verif.nc, output/data/runs/b30a-0209f019/202408030000/verif.nc, output/data/runs/b30a-0209f019/202408040000/verif.nc, output/data/runs/b30a-0209f019/202408050000/verif.nc, output/data/runs/b30a-0209f019/202408060000/verif.nc, output/data/runs/b30a-0209f019/202408070000/verif.nc, output/data/runs/b30a-0209f019/202408080000/verif.nc, output/data/runs/b30a-0209f019/202408090000/verif.nc, output/data/runs/b30a-0209f019/202408100000/verif.nc, output/data/runs/b30a-0209f019/202408110000/verif.nc, output/data/runs/b30a-0209f019/202408120000/verif.nc, output/data/runs/b30a-0209f019/202408130000/verif.nc, output/data/runs/b30a-0209f019/202408140000/verif.nc, output/data/runs/b30a-0209f019/202408150000/verif.nc, output/data/runs/b30a-0209f019/202408160000/verif.nc, output/data/runs/b30a-0209f019/202408170000/verif.nc, output/data/runs/b30a-0209f019/202408180000/verif.nc, output/data/runs/b30a-0209f019/202408190000/verif.nc, output/data/runs/b30a-0209f019/202408200000/verif.nc, output/data/runs/b30a-0209f019/202408210000/verif.nc, output/data/runs/b30a-0209f019/202408220000/verif.nc, output/data/runs/b30a-0209f019/202408230000/verif.nc, output/data/runs/b30a-0209f019/202408240000/verif.nc, output/data/runs/b30a-0209f019/202408250000/verif.nc, output/data/runs/b30a-0209f019/202408260000/verif.nc, output/data/runs/b30a-0209f019/202408270000/verif.nc, output/data/runs/b30a-0209f019/202408280000/verif.nc, output/data/runs/b30a-0209f019/202408290000/verif.nc, output/data/runs/b30a-0209f019/202408300000/verif.nc, output/data/runs/b30a-0209f019/202408310000/verif.nc, output/data/runs/b30a-0209f019/202409010000/verif.nc, output/data/runs/b30a-0209f019/202409020000/verif.nc, output/data/runs/b30a-0209f019/202409030000/verif.nc, output/data/runs/b30a-0209f019/202409040000/verif.nc, output/data/runs/b30a-0209f019/202409050000/verif.nc, output/data/runs/b30a-0209f019/202409060000/verif.nc, output/data/runs/b30a-0209f019/202409070000/verif.nc, output/data/runs/b30a-0209f019/202409080000/verif.nc, output/data/runs/b30a-0209f019/202409090000/verif.nc, output/data/runs/b30a-0209f019/202409100000/verif.nc, output/data/runs/b30a-0209f019/202409110000/verif.nc, output/data/runs/b30a-0209f019/202409120000/verif.nc, output/data/runs/b30a-0209f019/202409130000/verif.nc, output/data/runs/b30a-0209f019/202409140000/verif.nc, output/data/runs/b30a-0209f019/202409150000/verif.nc, output/data/runs/b30a-0209f019/202409160000/verif.nc, output/data/runs/b30a-0209f019/202409170000/verif.nc, output/data/runs/b30a-0209f019/202409180000/verif.nc, output/data/runs/b30a-0209f019/202409190000/verif.nc, output/data/runs/b30a-0209f019/202409200000/verif.nc, output/data/runs/b30a-0209f019/202409210000/verif.nc, output/data/runs/b30a-0209f019/202409220000/verif.nc, output/data/runs/b30a-0209f019/202409230000/verif.nc, output/data/runs/b30a-0209f019/202409240000/verif.nc, output/data/runs/b30a-0209f019/202409250000/verif.nc, output/data/runs/b30a-0209f019/202409260000/verif.nc, output/data/runs/b30a-0209f019/202409270000/verif.nc, output/data/runs/b30a-0209f019/202409280000/verif.nc, output/data/runs/b30a-0209f019/202409290000/verif.nc, output/data/runs/b30a-0209f019/202409300000/verif.nc, output/data/runs/b30a-0209f019/202410010000/verif.nc, output/data/runs/b30a-0209f019/202410020000/verif.nc, output/data/runs/b30a-0209f019/202410030000/verif.nc, output/data/runs/b30a-0209f019/202410040000/verif.nc, output/data/runs/b30a-0209f019/202410050000/verif.nc, output/data/runs/b30a-0209f019/202410060000/verif.nc, output/data/runs/b30a-0209f019/202410070000/verif.nc, output/data/runs/b30a-0209f019/202410080000/verif.nc, output/data/runs/b30a-0209f019/202410090000/verif.nc, output/data/runs/b30a-0209f019/202410100000/verif.nc, output/data/runs/b30a-0209f019/202410110000/verif.nc, output/data/runs/b30a-0209f019/202410120000/verif.nc, output/data/runs/b30a-0209f019/202410130000/verif.nc, output/data/runs/b30a-0209f019/202410140000/verif.nc, output/data/runs/b30a-0209f019/202410150000/verif.nc, output/data/runs/b30a-0209f019/202410160000/verif.nc, output/data/runs/b30a-0209f019/202410170000/verif.nc, output/data/runs/b30a-0209f019/202410180000/verif.nc, output/data/runs/b30a-0209f019/202410190000/verif.nc, output/data/runs/b30a-0209f019/202410200000/verif.nc, output/data/runs/b30a-0209f019/202410210000/verif.nc, output/data/runs/b30a-0209f019/202410220000/verif.nc, output/data/runs/b30a-0209f019/202410230000/verif.nc, output/data/runs/b30a-0209f019/202410240000/verif.nc, output/data/runs/b30a-0209f019/202410250000/verif.nc, output/data/runs/b30a-0209f019/202410260000/verif.nc, output/data/runs/b30a-0209f019/202410270000/verif.nc, output/data/runs/b30a-0209f019/202410280000/verif.nc, output/data/runs/b30a-0209f019/202410290000/verif.nc, output/data/runs/b30a-0209f019/202410300000/verif.nc, output/data/runs/b30a-0209f019/202410310000/verif.nc, output/data/runs/b30a-0209f019/202411010000/verif.nc, output/data/runs/b30a-0209f019/202411020000/verif.nc, output/data/runs/b30a-0209f019/202411030000/verif.nc, output/data/runs/b30a-0209f019/202411040000/verif.nc, output/data/runs/b30a-0209f019/202411050000/verif.nc, output/data/runs/b30a-0209f019/202411060000/verif.nc, output/data/runs/b30a-0209f019/202411070000/verif.nc, output/data/runs/b30a-0209f019/202411080000/verif.nc, output/data/runs/b30a-0209f019/202411090000/verif.nc, output/data/runs/b30a-0209f019/202411100000/verif.nc, output/data/runs/b30a-0209f019/202411110000/verif.nc, output/data/runs/b30a-0209f019/202411120000/verif.nc, output/data/runs/b30a-0209f019/202411130000/verif.nc, output/data/runs/b30a-0209f019/202411140000/verif.nc, output/data/runs/b30a-0209f019/202411150000/verif.nc, output/data/runs/b30a-0209f019/202411160000/verif.nc, output/data/runs/b30a-0209f019/202411170000/verif.nc, output/data/runs/b30a-0209f019/202411180000/verif.nc, output/data/runs/b30a-0209f019/202411190000/verif.nc, output/data/runs/b30a-0209f019/202411200000/verif.nc, output/data/runs/b30a-0209f019/202411210000/verif.nc, output/data/runs/b30a-0209f019/202411220000/verif.nc, output/data/runs/b30a-0209f019/202411230000/verif.nc, output/data/runs/b30a-0209f019/202411240000/verif.nc, output/data/runs/b30a-0209f019/202411250000/verif.nc, output/data/runs/b30a-0209f019/202411260000/verif.nc, output/data/runs/b30a-0209f019/202411270000/verif.nc, output/data/runs/b30a-0209f019/202411280000/verif.nc, output/data/runs/b30a-0209f019/202411290000/verif.nc, output/data/runs/b30a-0209f019/202411300000/verif.nc, output/data/runs/b30a-0209f019/202412010000/verif.nc, output/data/runs/b30a-0209f019/202412020000/verif.nc, output/data/runs/b30a-0209f019/202412030000/verif.nc, output/data/runs/b30a-0209f019/202412040000/verif.nc, output/data/runs/b30a-0209f019/202412050000/verif.nc, output/data/runs/b30a-0209f019/202412060000/verif.nc, output/data/runs/b30a-0209f019/202412070000/verif.nc, output/data/runs/b30a-0209f019/202412080000/verif.nc, output/data/runs/b30a-0209f019/202412090000/verif.nc, output/data/runs/b30a-0209f019/202412100000/verif.nc, output/data/runs/b30a-0209f019/202412110000/verif.nc, output/data/runs/b30a-0209f019/202412120000/verif.nc, output/data/runs/b30a-0209f019/202412130000/verif.nc, output/data/runs/b30a-0209f019/202412140000/verif.nc, output/data/runs/b30a-0209f019/202412150000/verif.nc, output/data/runs/b30a-0209f019/202412160000/verif.nc, output/data/runs/b30a-0209f019/202412170000/verif.nc, output/data/runs/b30a-0209f019/202412180000/verif.nc, output/data/runs/b30a-0209f019/202412190000/verif.nc, output/data/runs/b30a-0209f019/202412200000/verif.nc, output/data/runs/b30a-0209f019/202412210000/verif.nc, output/data/runs/b30a-0209f019/202412220000/verif.nc, output/data/runs/b30a-0209f019/202412230000/verif.nc, output/data/runs/b30a-0209f019/202412240000/verif.nc, output/data/runs/b30a-0209f019/202412250000/verif.nc - output: output/data/runs/b30a-0209f019/verif_aggregated.nc - log: output/logs/verif_metrics_aggregation/b30a-0209f019.log - jobid: 362 - reason: Input files updated by another job: output/data/runs/b30a-0209f019/202407280000/verif.nc, output/data/runs/b30a-0209f019/202401040000/verif.nc, output/data/runs/b30a-0209f019/202406040000/verif.nc, output/data/runs/b30a-0209f019/202410170000/verif.nc, output/data/runs/b30a-0209f019/202407060000/verif.nc, output/data/runs/b30a-0209f019/202410260000/verif.nc, output/data/runs/b30a-0209f019/202408090000/verif.nc, output/data/runs/b30a-0209f019/202407230000/verif.nc, output/data/runs/b30a-0209f019/202408060000/verif.nc, output/data/runs/b30a-0209f019/202407180000/verif.nc, output/data/runs/b30a-0209f019/202405030000/verif.nc, output/data/runs/b30a-0209f019/202408030000/verif.nc, output/data/runs/b30a-0209f019/202410060000/verif.nc, output/data/runs/b30a-0209f019/202411300000/verif.nc, output/data/runs/b30a-0209f019/202412050000/verif.nc, output/data/runs/b30a-0209f019/202411240000/verif.nc, output/data/runs/b30a-0209f019/202403290000/verif.nc, output/data/runs/b30a-0209f019/202408110000/verif.nc, output/data/runs/b30a-0209f019/202410280000/verif.nc, output/data/runs/b30a-0209f019/202404090000/verif.nc, output/data/runs/b30a-0209f019/202411170000/verif.nc, output/data/runs/b30a-0209f019/202406110000/verif.nc, output/data/runs/b30a-0209f019/202406090000/verif.nc, output/data/runs/b30a-0209f019/202410240000/verif.nc, output/data/runs/b30a-0209f019/202401100000/verif.nc, output/data/runs/b30a-0209f019/202410130000/verif.nc, output/data/runs/b30a-0209f019/202402060000/verif.nc, output/data/runs/b30a-0209f019/202401060000/verif.nc, output/data/runs/b30a-0209f019/202406050000/verif.nc, output/data/runs/b30a-0209f019/202404080000/verif.nc, output/data/runs/b30a-0209f019/202403220000/verif.nc, output/data/runs/b30a-0209f019/202401170000/verif.nc, output/data/runs/b30a-0209f019/202403030000/verif.nc, output/data/runs/b30a-0209f019/202408120000/verif.nc, output/data/runs/b30a-0209f019/202411180000/verif.nc, output/data/runs/b30a-0209f019/202402230000/verif.nc, output/data/runs/b30a-0209f019/202406070000/verif.nc, output/data/runs/b30a-0209f019/202409060000/verif.nc, output/data/runs/b30a-0209f019/202408150000/verif.nc, output/data/runs/b30a-0209f019/202412170000/verif.nc, output/data/runs/b30a-0209f019/202403170000/verif.nc, output/data/runs/b30a-0209f019/202402100000/verif.nc, output/data/runs/b30a-0209f019/202411220000/verif.nc, output/data/runs/b30a-0209f019/202409160000/verif.nc, output/data/runs/b30a-0209f019/202411280000/verif.nc, output/data/runs/b30a-0209f019/202409290000/verif.nc, output/data/runs/b30a-0209f019/202402240000/verif.nc, output/data/runs/b30a-0209f019/202408220000/verif.nc, output/data/runs/b30a-0209f019/202408270000/verif.nc, output/data/runs/b30a-0209f019/202405040000/verif.nc, output/data/runs/b30a-0209f019/202412010000/verif.nc, output/data/runs/b30a-0209f019/202402150000/verif.nc, output/data/runs/b30a-0209f019/202404030000/verif.nc, output/data/runs/b30a-0209f019/202405310000/verif.nc, output/data/runs/b30a-0209f019/202410120000/verif.nc, output/data/runs/b30a-0209f019/202402120000/verif.nc, output/data/runs/b30a-0209f019/202404040000/verif.nc, output/data/runs/b30a-0209f019/202411290000/verif.nc, output/data/runs/b30a-0209f019/202411030000/verif.nc, output/data/runs/b30a-0209f019/202406080000/verif.nc, output/data/runs/b30a-0209f019/202402250000/verif.nc, output/data/runs/b30a-0209f019/202405120000/verif.nc, output/data/runs/b30a-0209f019/202410290000/verif.nc, output/data/runs/b30a-0209f019/202410150000/verif.nc, output/data/runs/b30a-0209f019/202408130000/verif.nc, output/data/runs/b30a-0209f019/202408300000/verif.nc, output/data/runs/b30a-0209f019/202404010000/verif.nc, output/data/runs/b30a-0209f019/202404050000/verif.nc, output/data/runs/b30a-0209f019/202411110000/verif.nc, output/data/runs/b30a-0209f019/202408200000/verif.nc, output/data/runs/b30a-0209f019/202408050000/verif.nc, output/data/runs/b30a-0209f019/202405210000/verif.nc, output/data/runs/b30a-0209f019/202409260000/verif.nc, output/data/runs/b30a-0209f019/202405010000/verif.nc, output/data/runs/b30a-0209f019/202402200000/verif.nc, output/data/runs/b30a-0209f019/202403160000/verif.nc, output/data/runs/b30a-0209f019/202401070000/verif.nc, output/data/runs/b30a-0209f019/202409240000/verif.nc, output/data/runs/b30a-0209f019/202409010000/verif.nc, output/data/runs/b30a-0209f019/202402210000/verif.nc, output/data/runs/b30a-0209f019/202409080000/verif.nc, output/data/runs/b30a-0209f019/202406030000/verif.nc, output/data/runs/b30a-0209f019/202409040000/verif.nc, output/data/runs/b30a-0209f019/202408190000/verif.nc, output/data/runs/b30a-0209f019/202404210000/verif.nc, output/data/runs/b30a-0209f019/202402010000/verif.nc, output/data/runs/b30a-0209f019/202407220000/verif.nc, output/data/runs/b30a-0209f019/202404110000/verif.nc, output/data/runs/b30a-0209f019/202406290000/verif.nc, output/data/runs/b30a-0209f019/202411010000/verif.nc, output/data/runs/b30a-0209f019/202404230000/verif.nc, output/data/runs/b30a-0209f019/202411210000/verif.nc, output/data/runs/b30a-0209f019/202407080000/verif.nc, output/data/runs/b30a-0209f019/202412240000/verif.nc, output/data/runs/b30a-0209f019/202406150000/verif.nc, output/data/runs/b30a-0209f019/202409230000/verif.nc, output/data/runs/b30a-0209f019/202403310000/verif.nc, output/data/runs/b30a-0209f019/202406250000/verif.nc, output/data/runs/b30a-0209f019/202407150000/verif.nc, output/data/runs/b30a-0209f019/202410200000/verif.nc, output/data/runs/b30a-0209f019/202406160000/verif.nc, output/data/runs/b30a-0209f019/202402110000/verif.nc, output/data/runs/b30a-0209f019/202401020000/verif.nc, output/data/runs/b30a-0209f019/202403150000/verif.nc, output/data/runs/b30a-0209f019/202407030000/verif.nc, output/data/runs/b30a-0209f019/202402070000/verif.nc, output/data/runs/b30a-0209f019/202408240000/verif.nc, output/data/runs/b30a-0209f019/202407090000/verif.nc, output/data/runs/b30a-0209f019/202403040000/verif.nc, output/data/runs/b30a-0209f019/202406020000/verif.nc, output/data/runs/b30a-0209f019/202409070000/verif.nc, output/data/runs/b30a-0209f019/202410250000/verif.nc, output/data/runs/b30a-0209f019/202405220000/verif.nc, output/data/runs/b30a-0209f019/202409090000/verif.nc, output/data/runs/b30a-0209f019/202402040000/verif.nc, output/data/runs/b30a-0209f019/202405230000/verif.nc, output/data/runs/b30a-0209f019/202408290000/verif.nc, output/data/runs/b30a-0209f019/202407130000/verif.nc, output/data/runs/b30a-0209f019/202404250000/verif.nc, output/data/runs/b30a-0209f019/202406260000/verif.nc, output/data/runs/b30a-0209f019/202410100000/verif.nc, output/data/runs/b30a-0209f019/202411270000/verif.nc, output/data/runs/b30a-0209f019/202404270000/verif.nc, output/data/runs/b30a-0209f019/202405070000/verif.nc, output/data/runs/b30a-0209f019/202401140000/verif.nc, output/data/runs/b30a-0209f019/202412100000/verif.nc, output/data/runs/b30a-0209f019/202409140000/verif.nc, output/data/runs/b30a-0209f019/202403270000/verif.nc, output/data/runs/b30a-0209f019/202411080000/verif.nc, output/data/runs/b30a-0209f019/202409020000/verif.nc, output/data/runs/b30a-0209f019/202409030000/verif.nc, output/data/runs/b30a-0209f019/202411100000/verif.nc, output/data/runs/b30a-0209f019/202408170000/verif.nc, output/data/runs/b30a-0209f019/202403020000/verif.nc, output/data/runs/b30a-0209f019/202408100000/verif.nc, output/data/runs/b30a-0209f019/202403060000/verif.nc, output/data/runs/b30a-0209f019/202407110000/verif.nc, output/data/runs/b30a-0209f019/202403090000/verif.nc, output/data/runs/b30a-0209f019/202402260000/verif.nc, output/data/runs/b30a-0209f019/202402140000/verif.nc, output/data/runs/b30a-0209f019/202404180000/verif.nc, output/data/runs/b30a-0209f019/202401190000/verif.nc, output/data/runs/b30a-0209f019/202408260000/verif.nc, output/data/runs/b30a-0209f019/202405100000/verif.nc, output/data/runs/b30a-0209f019/202408160000/verif.nc, output/data/runs/b30a-0209f019/202402170000/verif.nc, output/data/runs/b30a-0209f019/202404220000/verif.nc, output/data/runs/b30a-0209f019/202405240000/verif.nc, output/data/runs/b30a-0209f019/202407310000/verif.nc, output/data/runs/b30a-0209f019/202404160000/verif.nc, output/data/runs/b30a-0209f019/202408310000/verif.nc, output/data/runs/b30a-0209f019/202404300000/verif.nc, output/data/runs/b30a-0209f019/202407250000/verif.nc, output/data/runs/b30a-0209f019/202408070000/verif.nc, output/data/runs/b30a-0209f019/202406140000/verif.nc, output/data/runs/b30a-0209f019/202407100000/verif.nc, output/data/runs/b30a-0209f019/202403080000/verif.nc, output/data/runs/b30a-0209f019/202407070000/verif.nc, output/data/runs/b30a-0209f019/202407210000/verif.nc, output/data/runs/b30a-0209f019/202403280000/verif.nc, output/data/runs/b30a-0209f019/202402020000/verif.nc, output/data/runs/b30a-0209f019/202410140000/verif.nc, output/data/runs/b30a-0209f019/202411160000/verif.nc, output/data/runs/b30a-0209f019/202405110000/verif.nc, output/data/runs/b30a-0209f019/202410070000/verif.nc, output/data/runs/b30a-0209f019/202404120000/verif.nc, output/data/runs/b30a-0209f019/202401290000/verif.nc, output/data/runs/b30a-0209f019/202411140000/verif.nc, output/data/runs/b30a-0209f019/202401110000/verif.nc, output/data/runs/b30a-0209f019/202412190000/verif.nc, output/data/runs/b30a-0209f019/202408230000/verif.nc, output/data/runs/b30a-0209f019/202403200000/verif.nc, output/data/runs/b30a-0209f019/202401200000/verif.nc, output/data/runs/b30a-0209f019/202405270000/verif.nc, output/data/runs/b30a-0209f019/202405250000/verif.nc, output/data/runs/b30a-0209f019/202402280000/verif.nc, output/data/runs/b30a-0209f019/202409210000/verif.nc, output/data/runs/b30a-0209f019/202412140000/verif.nc, output/data/runs/b30a-0209f019/202405170000/verif.nc, output/data/runs/b30a-0209f019/202402160000/verif.nc, output/data/runs/b30a-0209f019/202401210000/verif.nc, output/data/runs/b30a-0209f019/202406170000/verif.nc, output/data/runs/b30a-0209f019/202406010000/verif.nc, output/data/runs/b30a-0209f019/202405050000/verif.nc, output/data/runs/b30a-0209f019/202404130000/verif.nc, output/data/runs/b30a-0209f019/202401310000/verif.nc, output/data/runs/b30a-0209f019/202412080000/verif.nc, output/data/runs/b30a-0209f019/202403300000/verif.nc, output/data/runs/b30a-0209f019/202401080000/verif.nc, output/data/runs/b30a-0209f019/202412220000/verif.nc, output/data/runs/b30a-0209f019/202411260000/verif.nc, output/data/runs/b30a-0209f019/202409300000/verif.nc, output/data/runs/b30a-0209f019/202405190000/verif.nc, output/data/runs/b30a-0209f019/202405020000/verif.nc, output/data/runs/b30a-0209f019/202408210000/verif.nc, output/data/runs/b30a-0209f019/202401260000/verif.nc, output/data/runs/b30a-0209f019/202405140000/verif.nc, output/data/runs/b30a-0209f019/202401180000/verif.nc, output/data/runs/b30a-0209f019/202407260000/verif.nc, output/data/runs/b30a-0209f019/202411070000/verif.nc, output/data/runs/b30a-0209f019/202409110000/verif.nc, output/data/runs/b30a-0209f019/202403010000/verif.nc, output/data/runs/b30a-0209f019/202404060000/verif.nc, output/data/runs/b30a-0209f019/202404170000/verif.nc, output/data/runs/b30a-0209f019/202407290000/verif.nc, output/data/runs/b30a-0209f019/202409130000/verif.nc, output/data/runs/b30a-0209f019/202403070000/verif.nc, output/data/runs/b30a-0209f019/202404140000/verif.nc, output/data/runs/b30a-0209f019/202402180000/verif.nc, output/data/runs/b30a-0209f019/202410020000/verif.nc, output/data/runs/b30a-0209f019/202409150000/verif.nc, output/data/runs/b30a-0209f019/202407120000/verif.nc, output/data/runs/b30a-0209f019/202411040000/verif.nc, output/data/runs/b30a-0209f019/202409280000/verif.nc, output/data/runs/b30a-0209f019/202412020000/verif.nc, output/data/runs/b30a-0209f019/202406210000/verif.nc, output/data/runs/b30a-0209f019/202403100000/verif.nc, output/data/runs/b30a-0209f019/202406180000/verif.nc, output/data/runs/b30a-0209f019/202404290000/verif.nc, output/data/runs/b30a-0209f019/202401130000/verif.nc, output/data/runs/b30a-0209f019/202408250000/verif.nc, output/data/runs/b30a-0209f019/202402080000/verif.nc, output/data/runs/b30a-0209f019/202411250000/verif.nc, output/data/runs/b30a-0209f019/202409180000/verif.nc, output/data/runs/b30a-0209f019/202410270000/verif.nc, output/data/runs/b30a-0209f019/202411090000/verif.nc, output/data/runs/b30a-0209f019/202402270000/verif.nc, output/data/runs/b30a-0209f019/202410030000/verif.nc, output/data/runs/b30a-0209f019/202406240000/verif.nc, output/data/runs/b30a-0209f019/202408280000/verif.nc, output/data/runs/b30a-0209f019/202409050000/verif.nc, output/data/runs/b30a-0209f019/202404240000/verif.nc, output/data/runs/b30a-0209f019/202402130000/verif.nc, output/data/runs/b30a-0209f019/202407160000/verif.nc, output/data/runs/b30a-0209f019/202410160000/verif.nc, output/data/runs/b30a-0209f019/202401250000/verif.nc, output/data/runs/b30a-0209f019/202405090000/verif.nc, output/data/runs/b30a-0209f019/202410010000/verif.nc, output/data/runs/b30a-0209f019/202410210000/verif.nc, output/data/runs/b30a-0209f019/202403120000/verif.nc, output/data/runs/b30a-0209f019/202401280000/verif.nc, output/data/runs/b30a-0209f019/202404150000/verif.nc, output/data/runs/b30a-0209f019/202403240000/verif.nc, output/data/runs/b30a-0209f019/202403210000/verif.nc, output/data/runs/b30a-0209f019/202411050000/verif.nc, output/data/runs/b30a-0209f019/202406230000/verif.nc, output/data/runs/b30a-0209f019/202403260000/verif.nc, output/data/runs/b30a-0209f019/202409200000/verif.nc, output/data/runs/b30a-0209f019/202410110000/verif.nc, output/data/runs/b30a-0209f019/202404200000/verif.nc, output/data/runs/b30a-0209f019/202401030000/verif.nc, output/data/runs/b30a-0209f019/202405130000/verif.nc, output/data/runs/b30a-0209f019/202407170000/verif.nc, output/data/runs/b30a-0209f019/202406100000/verif.nc, output/data/runs/b30a-0209f019/202410310000/verif.nc, output/data/runs/b30a-0209f019/202401240000/verif.nc, output/data/runs/b30a-0209f019/202402030000/verif.nc, output/data/runs/b30a-0209f019/202409220000/verif.nc, output/data/runs/b30a-0209f019/202403230000/verif.nc, output/data/runs/b30a-0209f019/202410230000/verif.nc, output/data/runs/b30a-0209f019/202405290000/verif.nc, output/data/runs/b30a-0209f019/202408180000/verif.nc, output/data/runs/b30a-0209f019/202402090000/verif.nc, output/data/runs/b30a-0209f019/202405200000/verif.nc, output/data/runs/b30a-0209f019/202403110000/verif.nc, output/data/runs/b30a-0209f019/202403140000/verif.nc, output/data/runs/b30a-0209f019/202406190000/verif.nc, output/data/runs/b30a-0209f019/202412160000/verif.nc, output/data/runs/b30a-0209f019/202412230000/verif.nc, output/data/runs/b30a-0209f019/202406060000/verif.nc, output/data/runs/b30a-0209f019/202410050000/verif.nc, output/data/runs/b30a-0209f019/202409270000/verif.nc, output/data/runs/b30a-0209f019/202401160000/verif.nc, output/data/runs/b30a-0209f019/202411020000/verif.nc, output/data/runs/b30a-0209f019/202410220000/verif.nc, output/data/runs/b30a-0209f019/202412200000/verif.nc, output/data/runs/b30a-0209f019/202410190000/verif.nc, output/data/runs/b30a-0209f019/202404190000/verif.nc, output/data/runs/b30a-0209f019/202410300000/verif.nc, output/data/runs/b30a-0209f019/202405180000/verif.nc, output/data/runs/b30a-0209f019/202403190000/verif.nc, output/data/runs/b30a-0209f019/202412130000/verif.nc, output/data/runs/b30a-0209f019/202403180000/verif.nc, output/data/runs/b30a-0209f019/202411190000/verif.nc, output/data/runs/b30a-0209f019/202405060000/verif.nc, output/data/runs/b30a-0209f019/202402220000/verif.nc, output/data/runs/b30a-0209f019/202410080000/verif.nc, output/data/runs/b30a-0209f019/202412250000/verif.nc, output/data/runs/b30a-0209f019/202410180000/verif.nc, output/data/runs/b30a-0209f019/202404020000/verif.nc, output/data/runs/b30a-0209f019/202404100000/verif.nc, output/data/runs/b30a-0209f019/202408010000/verif.nc, output/data/runs/b30a-0209f019/202401050000/verif.nc, output/data/runs/b30a-0209f019/202404280000/verif.nc, output/data/runs/b30a-0209f019/202406300000/verif.nc, output/data/runs/b30a-0209f019/202411150000/verif.nc, output/data/runs/b30a-0209f019/202407300000/verif.nc, output/data/runs/b30a-0209f019/202412090000/verif.nc, output/data/runs/b30a-0209f019/202409170000/verif.nc, output/data/runs/b30a-0209f019/202405260000/verif.nc, output/data/runs/b30a-0209f019/202411060000/verif.nc, output/data/runs/b30a-0209f019/202408140000/verif.nc, output/data/runs/b30a-0209f019/202412150000/verif.nc, output/data/runs/b30a-0209f019/202408020000/verif.nc, output/data/runs/b30a-0209f019/202412070000/verif.nc, output/data/runs/b30a-0209f019/202401230000/verif.nc, output/data/runs/b30a-0209f019/202411130000/verif.nc, output/data/runs/b30a-0209f019/202407140000/verif.nc, output/data/runs/b30a-0209f019/202409120000/verif.nc, output/data/runs/b30a-0209f019/202409250000/verif.nc, output/data/runs/b30a-0209f019/202401150000/verif.nc, output/data/runs/b30a-0209f019/202412180000/verif.nc, output/data/runs/b30a-0209f019/202412060000/verif.nc, output/data/runs/b30a-0209f019/202412040000/verif.nc, output/data/runs/b30a-0209f019/202411230000/verif.nc, output/data/runs/b30a-0209f019/202404260000/verif.nc, output/data/runs/b30a-0209f019/202407050000/verif.nc, output/data/runs/b30a-0209f019/202403250000/verif.nc, output/data/runs/b30a-0209f019/202402050000/verif.nc, output/data/runs/b30a-0209f019/202407040000/verif.nc, output/data/runs/b30a-0209f019/202406270000/verif.nc, output/data/runs/b30a-0209f019/202402290000/verif.nc, output/data/runs/b30a-0209f019/202406130000/verif.nc, output/data/runs/b30a-0209f019/202406220000/verif.nc, output/data/runs/b30a-0209f019/202407020000/verif.nc, output/data/runs/b30a-0209f019/202403050000/verif.nc, output/data/runs/b30a-0209f019/202408040000/verif.nc, output/data/runs/b30a-0209f019/202406120000/verif.nc, output/data/runs/b30a-0209f019/202411120000/verif.nc, output/data/runs/b30a-0209f019/202407240000/verif.nc, output/data/runs/b30a-0209f019/202412030000/verif.nc, output/data/runs/b30a-0209f019/202410040000/verif.nc, output/data/runs/b30a-0209f019/202401300000/verif.nc, output/data/runs/b30a-0209f019/202412110000/verif.nc, output/data/runs/b30a-0209f019/202407270000/verif.nc, output/data/runs/b30a-0209f019/202405300000/verif.nc, output/data/runs/b30a-0209f019/202407190000/verif.nc, output/data/runs/b30a-0209f019/202406280000/verif.nc, output/data/runs/b30a-0209f019/202410090000/verif.nc, output/data/runs/b30a-0209f019/202401220000/verif.nc, output/data/runs/b30a-0209f019/202401090000/verif.nc, output/data/runs/b30a-0209f019/202409100000/verif.nc, output/data/runs/b30a-0209f019/202408080000/verif.nc, output/data/runs/b30a-0209f019/202406200000/verif.nc, output/data/runs/b30a-0209f019/202407010000/verif.nc, output/data/runs/b30a-0209f019/202403130000/verif.nc, output/data/runs/b30a-0209f019/202401270000/verif.nc, output/data/runs/b30a-0209f019/202412210000/verif.nc, output/data/runs/b30a-0209f019/202405150000/verif.nc, output/data/runs/b30a-0209f019/202409190000/verif.nc, output/data/runs/b30a-0209f019/202407200000/verif.nc, output/data/runs/b30a-0209f019/202405280000/verif.nc, output/data/runs/b30a-0209f019/202401120000/verif.nc, output/data/runs/b30a-0209f019/202404070000/verif.nc, output/data/runs/b30a-0209f019/202402190000/verif.nc, output/data/runs/b30a-0209f019/202405080000/verif.nc, output/data/runs/b30a-0209f019/202412120000/verif.nc, output/data/runs/b30a-0209f019/202405160000/verif.nc, output/data/runs/b30a-0209f019/202411200000/verif.nc - wildcards: run_id=b30a-0209f019 - resources: mem_mb=250000, mem_mib=238419, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=1440, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 -Would remove temporary output output/data/runs/b30a-0209f019/202401020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202401310000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202402290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202403310000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202404300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202405310000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202406300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202407310000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202408310000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202409300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202410310000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411250000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411260000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411270000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411280000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411290000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202411300000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412010000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412020000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412030000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412040000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412050000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412060000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412070000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412080000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412090000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412100000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412110000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412120000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412130000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412140000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412150000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412160000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412170000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412180000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412190000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412200000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412210000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412220000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412230000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412240000/verif.nc -Would remove temporary output output/data/runs/b30a-0209f019/202412250000/verif.nc - -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_JJA_6.png - jobid: 1448 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_JJA_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=JJA, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule verif_metrics_plot: - input: workflow/scripts/verif_plot_metrics.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/plots - log: output/logs/verif_metrics_plot/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd.log - jobid: 1443 - reason: Missing output files: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/plots; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd - resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=16, mem_mb_per_cpu=1800, runtime=20 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_all_6.png - jobid: 1450 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_all_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=all, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_all_6.png - jobid: 1445 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_all_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=all, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_SON_6.png - jobid: 1454 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_SON_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=SON, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_DJF_6.png - jobid: 1446 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_DJF_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=DJF, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_MAM_6.png - jobid: 1447 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_MAM_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=MAM, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule report_experiment_dashboard: - input: workflow/scripts/report_experiment_dashboard.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc, output/data/runs/b30a-0209f019/verif_aggregated.nc, resources/report/dashboard/template.html.jinja2, resources/report/dashboard/script.js, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/config/forecasters-ich1_mod_02_1yr.yaml - output: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/dashboard - log: output/logs/report_experiment_dashboard/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd.log - jobid: 1 - reason: Missing output files: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/dashboard; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc - wildcards: experiment=experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_SON_6.png - jobid: 1449 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_SON_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=SON, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_JJA_6.png - jobid: 1453 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_JJA_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=JJA, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_DJF_6.png - jobid: 1451 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_DJF_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=DJF, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule plot_summary_stat_maps: - input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc - output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_MAM_6.png - jobid: 1452 - reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_MAM_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc - wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=MAM, leadtime=6 - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 -[Thu Feb 26 12:18:24 2026] -rule experiment_all: - input: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/dashboard, output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/plots, output/data/runs/b30a-0209f019/summary.md, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_all_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_DJF_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_MAM_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_JJA_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_SON_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_all_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_DJF_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_MAM_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_JJA_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_SON_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_all_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_DJF_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_MAM_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_JJA_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_SON_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_all_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_DJF_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_MAM_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_JJA_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_SON_6.png - jobid: 0 - reason: Input files updated by another job: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_JJA_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_DJF_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_MAM_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_SON_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_SON_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_MAM_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_DJF_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_all_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_DJF_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_all_6.png, output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/dashboard, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_JJA_6.png, output/data/runs/b30a-0209f019/summary.md, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_DJF_6.png, output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/plots, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_SON_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_JJA_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_MAM_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_SON_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_MAM_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_all_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_all_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_JJA_6.png - resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= -Job stats: -job count ----------------------------------- ------- -execute_inference 359 -experiment_all 1 -make_squashfs_image 1 -plot_summary_stat_maps 10 -plot_summary_stat_maps_baseline 10 -prepare_inference_forecaster 358 -report_experiment_dashboard 1 -verif_metrics 359 -verif_metrics_aggregation 1 -verif_metrics_aggregation_baseline 1 -verif_metrics_baseline 359 -verif_metrics_plot 1 -write_summary 1 -total 1462 - -Reasons: - (check individual jobs above for details) - code has changed since last execution: - prepare_inference_forecaster - input files updated by another job: - execute_inference, experiment_all, plot_summary_stat_maps, plot_summary_stat_maps_baseline, report_experiment_dashboard, verif_metrics, verif_metrics_aggregation, verif_metrics_aggregation_baseline, verif_metrics_plot - output files have to be generated: - make_squashfs_image, plot_summary_stat_maps, plot_summary_stat_maps_baseline, report_experiment_dashboard, verif_metrics, verif_metrics_baseline, verif_metrics_plot - set of input files has changed since last execution: - write_summary - updated input files: - make_squashfs_image, verif_metrics_baseline -Some jobs were triggered by provenance information, see 'reason' section in the rule displays above. -If you prefer that only modification time is used to determine whether a job shall be executed, use the command line option '--rerun-triggers mtime' (also see --help). -If you are sure that a change for a certain output file (say, ) won't change the result (e.g. because you just changed the formatting of a script or environment definition), you can also wipe its metadata to skip such a trigger via 'snakemake --cleanup-metadata '. -Rules with provenance triggered jobs: prepare_inference_forecaster write_summary - -This was a dry-run (flag -n). The order of jobs does not reflect the order of execution. From 85b18632c98096d094c2e9946965ebf91e95d9c8 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 11:10:52 +0200 Subject: [PATCH 080/129] Cleanup: drop dead params block - Remove obsolete nc_out_dir params from plot_summary_stat_maps_baseline (pointed at the wrong file, never used in the shell command). --- workflow/rules/plot.smk | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index dc20bacd..0942cecb 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -178,9 +178,4 @@ use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: output: OUT_ROOT / "results/{experiment}/metric_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", log: - OUT_ROOT / "logs/plot_summary_stat_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", - params: - nc_out_dir=lambda wc: ( - Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" - # not sure if this is actually needed. Verification file is already specified above as input. Leave it for the time being. - ).resolve() \ No newline at end of file + OUT_ROOT / "logs/plot_summary_stat_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", \ No newline at end of file From 4e312f38e177595e314ba9a182771a307de1de39 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 11:16:39 +0200 Subject: [PATCH 081/129] Untrack dev-only example configs These were retained as personal development configs but shouldn't be part of the shared repo. Drop from tracking; local copies remain on disk as untracked files. --- config/forecasters-co2-mod-01.yaml | 78 ----------------- ...l_adj_lfr_05_Stage_E_only_default_exp.yaml | 87 ------------------- config/forecasters-ich1_mod_01.yaml | 79 ----------------- config/forecasters-ich1_mod_02_1yr.yaml | 79 ----------------- 4 files changed, 323 deletions(-) delete mode 100644 config/forecasters-co2-mod-01.yaml delete mode 100644 config/forecasters-ich1-oper-small_adj_lfr_05_Stage_E_only_default_exp.yaml delete mode 100644 config/forecasters-ich1_mod_01.yaml delete mode 100644 config/forecasters-ich1_mod_02_1yr.yaml diff --git a/config/forecasters-co2-mod-01.yaml b/config/forecasters-co2-mod-01.yaml deleted file mode 100644 index c551a118..00000000 --- a/config/forecasters-co2-mod-01.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# yaml-language-server: $schema=../workflow/tools/config.schema.json -description: | - Evaluate skill of COSMO-E emulator (M-1 forecaster). - -dates: - start: 2020-01-02T00:00 - end: 2020-01-31T00:00 - frequency: 24h - -runs: - - forecaster: - mlflow_id: d0846032fc7248a58b089cbe8fa4c511 - label: M-1 forecaster - steps: 0/120/6 - config: resources/inference/configs/sgm-forecaster-global_trimedge.yaml - extra_dependencies: - - git+https://github.com/ecmwf/anemoi-inference.git@0.8.3 - -baselines: - - baseline: - baseline_id: COSMO-E - label: COSMO-E - root: /store_new/mch/msopr/ml/COSMO-E - steps: 0/120/6 - -analysis: - label: COSMO KENDA - analysis_zarr: /scratch/mch/fzanetta/data/anemoi/datasets/mch-co2-an-archive-0p02-2015-2020-6h-v3-pl13.zarr - -stratification: - regions: - - jura - - mittelland - - voralpen - - alpennordhang - - innerealpentaeler - - alpensuedseite - root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 - -locations: - output_root: output/ - mlflow_uri: - - https://servicedepl.meteoswiss.ch/mlstore - - https://mlflow.ecmwf.int - -profile: - executor: slurm - global_resources: - gpus: 16 - default_resources: - slurm_partition: "postproc" - cpus_per_task: 1 - mem_mb_per_cpu: 1800 - runtime: "23h" - gpus: 0 - jobs: 50 - -spatial_verification: - params: - - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC - leadtimes: - - 6 - - 24 - metrics: - - BIAS - - RMSE - regions: - - switzerland - - centraleurope - seasons: - - all diff --git a/config/forecasters-ich1-oper-small_adj_lfr_05_Stage_E_only_default_exp.yaml b/config/forecasters-ich1-oper-small_adj_lfr_05_Stage_E_only_default_exp.yaml deleted file mode 100644 index d9051a94..00000000 --- a/config/forecasters-ich1-oper-small_adj_lfr_05_Stage_E_only_default_exp.yaml +++ /dev/null @@ -1,87 +0,0 @@ -# yaml-language-server: $schema=../workflow/tools/config.schema.json -description: | - Evaluate skill of ICON-CH1 forecasters from experiment 602 (excluding stage_B, AE, multi_dataset_icon_era5). - -dates: - start: 2025-03-01T00:00 - end: 2025-05-31T00:00 - frequency: 24h - - -runs: - - - forecaster: - inference_resources: - slurm_partition: normal-shared - checkpoint: https://service.meteoswiss.ch/mlstore#/experiments/602/runs/fd63e17043014af59170c7beca516b95 - label: stage_E_realch1 - steps: 0/120/6 - config: resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml - extra_requirements: - - git+https://github.com/ecmwf/anemoi-inference.git@b9aaee5df86614cad9d8d08b76876a4be4e980db - - -baselines: - - baseline: - baseline_id: ICON-CH1-EPS - label: ICON-CH1-ctrl - root: /scratch/mch/cmerker/ICON-CH1-EPS - steps: 0/33/6 - - baseline: - baseline_id: ICON-CH2-EPS - label: ICON-CH2-ctrl - root: /scratch/mch/cmerker/ICON-CH2-EPS - steps: 0/120/6 - -truth: - label: KENDA-CH1 - root: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2025-1h-pl13-v1.0.zarr - -stratification: - regions: - - jura - - mittelland - - voralpen - - alpennordhang - - innerealpentaeler - - alpensuedseite - root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 - -locations: - output_root: /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/xyz_abc/evalml/output/ - -profile: - executor: slurm - global_resources: - gpus: 16 - default_resources: - slurm_partition: "postproc" - cpus_per_task: 1 - mem_mb_per_cpu: 1800 - runtime: "1h" - gpus: 0 - jobs: 50 - batch_rules: - plot_forecast_frame: 32 - -spatial_verification: - params: - - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC - leadtimes: - - 6 - - 24 - metrics: - - BIAS - - RMSE - regions: - - switzerland - - centraleurope - seasons: - - all diff --git a/config/forecasters-ich1_mod_01.yaml b/config/forecasters-ich1_mod_01.yaml deleted file mode 100644 index 91110a02..00000000 --- a/config/forecasters-ich1_mod_01.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# yaml-language-server: $schema=../workflow/tools/config.schema.json -description: | - Evaluate skill of ICON-CH1 single. - -dates: - start: 2024-01-02T00:00 - end: 2024-01-31T00:00 - frequency: 24h - -runs: - - forecaster: - mlflow_id: b30acf68520a4bbd8324c44666561696 - label: stage_C_icon_1km - steps: 0/120/6 - config: resources/inference/configs/sgm-forecaster-global-ich1.yaml - disable_local_eccodes_definitions: true - extra_dependencies: - - git+https://github.com/ecmwf/anemoi-inference.git@main - -baselines: -- baseline: - baseline_id: ICON-CH1-EPS - label: ICON-CH1-EPS - root: /store_new/mch/msopr/ml/ICON-CH1-EPS - steps: 0/33/6 - -analysis: - label: REA-L-CH1 - analysis_zarr: /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - -stratification: - regions: - - jura - - mittelland - - voralpen - - alpennordhang - - innerealpentaeler - - alpensuedseite - root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 - -locations: - output_root: output/ - mlflow_uri: - - https://servicedepl.meteoswiss.ch/mlstore - - https://mlflow.ecmwf.int - -profile: - executor: slurm - global_resources: - gpus: 16 - default_resources: - slurm_partition: "postproc" - cpus_per_task: 1 - mem_mb_per_cpu: 1800 - runtime: "23h" - gpus: 0 - jobs: 50 - -spatial_verification: - params: - - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC - leadtimes: - - 6 - - 24 - metrics: - - BIAS - - RMSE - regions: - - switzerland - - centraleurope - seasons: - - all diff --git a/config/forecasters-ich1_mod_02_1yr.yaml b/config/forecasters-ich1_mod_02_1yr.yaml deleted file mode 100644 index 52615de1..00000000 --- a/config/forecasters-ich1_mod_02_1yr.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# yaml-language-server: $schema=../workflow/tools/config.schema.json -description: | - Evaluate skill of ICON-CH1 single. - -dates: - start: 2024-01-02T00:00 - end: 2024-12-25T00:00 - frequency: 24h - -runs: - - forecaster: - mlflow_id: b30acf68520a4bbd8324c44666561696 - label: stage_C_icon_1km - steps: 0/120/6 - config: resources/inference/configs/sgm-forecaster-global-ich1.yaml - disable_local_eccodes_definitions: true - extra_dependencies: - - git+https://github.com/ecmwf/anemoi-inference.git@main - -baselines: -- baseline: - baseline_id: ICON-CH1-EPS - label: ICON-CH1-EPS - root: /store_new/mch/msopr/ml/ICON-CH1-EPS - steps: 0/33/6 - -analysis: - label: REA-L-CH1 - analysis_zarr: /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr - -stratification: - regions: - - jura - - mittelland - - voralpen - - alpennordhang - - innerealpentaeler - - alpensuedseite - root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 - -locations: - output_root: output/ - mlflow_uri: - - https://servicedepl.meteoswiss.ch/mlstore - - https://mlflow.ecmwf.int - -profile: - executor: slurm - global_resources: - gpus: 16 - default_resources: - slurm_partition: "postproc" - cpus_per_task: 1 - mem_mb_per_cpu: 1800 - runtime: "23h" - gpus: 0 - jobs: 50 - -spatial_verification: - params: - - T_2M - - TD_2M - - U_10M - - V_10M - - SP_10M - - PS - - PMSL - - TOT_PREC - leadtimes: - - 6 - - 24 - metrics: - - BIAS - - RMSE - regions: - - switzerland - - centraleurope - seasons: - - all From 223ef7dc0d32de17763957320744209cf6f1fe46 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 11:19:11 +0200 Subject: [PATCH 082/129] Cleanup --- config/forecasters-ich1-oper-fixed.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index 54ce33e9..3dfbea5a 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -7,8 +7,8 @@ dates: # end: 2025-06-20T00:00 # frequency: 54h # or - # - 2024-01-01T12:00 - # - 2024-02-01T18:00 + - 2024-01-01T12:00 + - 2024-02-01T18:00 - 2025-03-01T00:00 From 467463969f97e47de9c633f1511aa6d370020c80 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 11:23:25 +0200 Subject: [PATCH 083/129] Cleanup --- src/plotting/colormap_defaults.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 5cb2ea03..6ab9622d 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -7,10 +7,12 @@ from matplotlib.colors import BoundaryNorm import numpy as np + def _fallback(): warnings.warn("No colormap found for this parameter, using fallback.", UserWarning) return {"cmap": plt.get_cmap("viridis"), "norm": None, "units": ""} + _CMAP_DEFAULTS = { "SP": {"cmap": plt.get_cmap("coolwarm", 11), "vmin": 800 * 100, "vmax": 1100 * 100}, "TD_2M": load_ncl_colormap("t2m_29lev.ct"), From bc2571df1e6676d0c48ac04ec799701b6d339806 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 11:29:45 +0200 Subject: [PATCH 084/129] More Cleanup --- src/verification/__init__.py | 3 +-- workflow/rules/verif.smk | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index 2a1f0c37..1b658fe9 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -78,13 +78,12 @@ def _compute_scores( dim: list[str], prefix="", suffix="", - source="" + source="", ) -> xr.Dataset: """ Compute basic verification metrics between two xarray DataArrays (fcst and obs). Returns a xarray Dataset with the computed metrics. """ - error = fcst - obs if dim == []: scores = xr.Dataset( diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index 41f6254e..38c0e869 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -97,7 +97,6 @@ rule verif_metrics: """ - def _restrict_reftimes_to_hours(reftimes, hours=None): """Restrict the reference times to specific hours.""" if hours is None: @@ -120,7 +119,7 @@ rule verif_metrics_aggregation: resources: cpus_per_task=24, mem_mb=250_000, - runtime="24h", + runtime="2h", slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ From 74d1548fd37d2d581fdc8bb63a0ddcd8f875dc03 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 11:34:54 +0200 Subject: [PATCH 085/129] More Cleanup --- src/plotting/compat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plotting/compat.py b/src/plotting/compat.py index 93634aa3..5ce97d3b 100644 --- a/src/plotting/compat.py +++ b/src/plotting/compat.py @@ -91,4 +91,3 @@ def load_state_from_raw( if key.startswith("field_"): state["fields"][key.removeprefix("field_")] = value return state - From 6b39d804ddad17fc56cbab57eb89f6f910641acc Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 11:43:02 +0200 Subject: [PATCH 086/129] More Cleanup ... --- src/plotting/compat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plotting/compat.py b/src/plotting/compat.py index 5ce97d3b..665287e0 100644 --- a/src/plotting/compat.py +++ b/src/plotting/compat.py @@ -5,7 +5,6 @@ import geopandas as gpd import numpy as np import pandas as pd -import xarray as xr from meteodatalab import data_source from meteodatalab import grib_decoder from shapely.geometry import MultiPoint From 51bf6d6bbdf21590cd129406e017fcbc1ca62cbd Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 11:57:29 +0200 Subject: [PATCH 087/129] Improved Domains --- src/plotting/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plotting/__init__.py b/src/plotting/__init__.py index c905fcc3..2148dd74 100644 --- a/src/plotting/__init__.py +++ b/src/plotting/__init__.py @@ -32,9 +32,10 @@ # The domains which are originally called "centraleurope" and "switzerland" # are mostly the same. I suggest making domain "switzerland" much smaller, # so that more spatial detail can be seen, especially in the complex - # topography of the alps. + # topography of the alps. Also, make domain "centraleurope" slightly smaller, + # to zoom in more on the LAM area, so that less plotting area outside is lost. "centraleurope": { - "extent": [-2.6, 19.5, 40.2, 52.3], + "extent": [-1.5, 18, 41.5, 51], "projection": _PROJECTIONS["orthographic"], }, "switzerland": { From 3151447fd4e760824ccc53aa25c4e7aede2178be Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 16:21:08 +0200 Subject: [PATCH 088/129] Tidy plot_summary_stat_maps.mo.py: drop dead code and dev comments Remove dev-commentary comments in the import cell, sever the unused `var` cross-cell dependency, and drop a trailing empty marimo cell. No behaviour change. --- workflow/scripts/plot_summary_stat_maps.mo.py | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index 9a560330..e1cca6ca 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -6,25 +6,17 @@ @app.cell def _(): - - # this sure stays the same. import logging from argparse import ArgumentParser from pathlib import Path - # this sure stays the same. import cartopy.crs as ccrs import earthkit.plots as ekp import numpy as np import xarray as xr - # this stays the same as well. from plotting import DOMAINS - - # no changes to StatePlotter required according to ChatGPT. from plotting import StatePlotter - - # Added some new colour maps for the Bias / MAE / RMSE map plots. from plotting.colormap_defaults import CMAP_DEFAULTS return ( ArgumentParser, @@ -100,8 +92,7 @@ def _(LOG, metric, param, season, verif_file, xr): ds = ds[var].sel(season=season) LOG.info("Selected DataArray: dims=%s, shape=%s, dtype=%s", ds.dims, ds.shape, ds.dtype) LOG.info("Value range: min=%.4g, max=%.4g, n_nan=%d", float(ds.min()), float(ds.max()), int(ds.isnull().sum())) - ds - return ds, var + return (ds,) @app.cell def _(CMAP_DEFAULTS, ekp): @@ -144,7 +135,6 @@ def _( param, region, season, - var, ): # plot individual fields import matplotlib.pyplot as plt @@ -178,15 +168,6 @@ def _( subplot.ax.legend(handles=[grey_patch], loc="lower left", fontsize=8) else: plotter.plot_field(subplot, plot_vals, **style_kwargs) - # subplot.ax.add_geometries( - # state["lam_envelope"], - # edgecolor="black", - # facecolor="none", - # crs=ccrs.PlateCarree(), - # ) - - # validtime = state["valid_time"].strftime("%Y%m%d%H%M") - # # leadtime = int(state["lead_time"].total_seconds() // 3600) fig.title(f"{metric} of {param}, Season: {season}, Lead Time: {lead_time}") @@ -195,10 +176,5 @@ def _( return -@app.cell -def _(): - return - - if __name__ == "__main__": app.run() From 9293f3750babf8b15cc8e5118b433e5cf41ab744 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 17:13:46 +0200 Subject: [PATCH 089/129] Black coast lines and country borders for better visibility --- workflow/scripts/plot_summary_stat_maps.mo.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index e1cca6ca..c7f9e32f 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -169,6 +169,12 @@ def _( else: plotter.plot_field(subplot, plot_vals, **style_kwargs) + # black coast lines and country borders for better visibility + # grey is hardly visible, especially when the shading colours are intense. + subplot.coastlines(edgecolor="black", linewidth=1.0, zorder=5) + subplot.borders(edgecolor="black", linewidth=0.5, zorder=5) + + fig.title(f"{metric} of {param}, Season: {season}, Lead Time: {lead_time}") fig.save(outfn, bbox_inches="tight", dpi=200) From 3a2e7adab4ad5ce76ac0d927af6e32080fa38f18 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 17:14:17 +0200 Subject: [PATCH 090/129] Drop hardcoded SLURM node-exclude lines These were workarounds for transient node issues at development time and shouldn't be baked into the workflow. --- workflow/rules/plot.smk | 1 - workflow/rules/verif.smk | 5 ----- 2 files changed, 6 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 0942cecb..87ff2960 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -156,7 +156,6 @@ rule plot_summary_stat_maps: slurm_partition="postproc", cpus_per_task=1, runtime="10m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index 38c0e869..5596a771 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -34,7 +34,6 @@ rule verif_metrics_baseline: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -82,7 +81,6 @@ rule verif_metrics: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -120,7 +118,6 @@ rule verif_metrics_aggregation: cpus_per_task=24, mem_mb=250_000, runtime="2h", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} {input.verif_nc} --output {output} > {log} 2>&1 @@ -190,7 +187,6 @@ rule verif_metrics_maps: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -222,7 +218,6 @@ rule verif_metrics_maps_baseline: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ From 04022f819e649ec2b6e9ada6fc69c4930b686839 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 17:31:55 +0200 Subject: [PATCH 091/129] Revert "Drop hardcoded SLURM node-exclude lines" This reverts commit 3a2e7adab4ad5ce76ac0d927af6e32080fa38f18. --- workflow/rules/plot.smk | 1 + workflow/rules/verification.smk | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index ecc9a3cb..28ae9022 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -159,6 +159,7 @@ rule plot_summary_stat_maps: slurm_partition="postproc", cpus_per_task=1, runtime="10m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 0df4743b..9f182ad3 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -34,6 +34,7 @@ rule verification_metrics_baseline: cpus_per_task=24, mem_mb=50_000, runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -81,6 +82,7 @@ rule verification_metrics: cpus_per_task=24, mem_mb=50_000, runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -118,6 +120,7 @@ rule verification_metrics_aggregation: cpus_per_task=24, mem_mb=250_000, runtime="2h", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} {input.verif_nc} --output {output} > {log} 2>&1 @@ -187,6 +190,7 @@ rule verification_metrics_maps: cpus_per_task=24, mem_mb=50_000, runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -218,6 +222,7 @@ rule verification_metrics_maps_baseline: cpus_per_task=24, mem_mb=50_000, runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ From 4d1aa6523f018b90fdc204d3b218d98fc94597fa Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 17:38:51 +0200 Subject: [PATCH 092/129] Apply pre-commit auto-fixes (whitespace, formatting, linting) --- src/evalml/cli.py | 11 +- src/plotting/__init__.py | 8 +- src/plotting/colormap_defaults.py | 155 ++++++++++++++---- workflow/Snakefile | 20 +-- workflow/rules/plot.smk | 18 +- workflow/rules/verification.smk | 15 +- workflow/scripts/plot_summary_stat_maps.mo.py | 50 ++++-- workflow/scripts/verification_metric_maps.py | 132 ++++++++++----- workflow/scripts/verification_metrics.py | 2 +- 9 files changed, 298 insertions(+), 113 deletions(-) diff --git a/src/evalml/cli.py b/src/evalml/cli.py index f344e35a..1c0fdfc7 100644 --- a/src/evalml/cli.py +++ b/src/evalml/cli.py @@ -172,7 +172,16 @@ def cli(): ) @workflow_options def experiment( - configfile, maps, cores, verbose, dry_run, unlock, report, dag, rulegraph, extra_smk_args + configfile, + maps, + cores, + verbose, + dry_run, + unlock, + report, + dag, + rulegraph, + extra_smk_args, ): execute_workflow( configfile, diff --git a/src/plotting/__init__.py b/src/plotting/__init__.py index 2148dd74..80d54b47 100644 --- a/src/plotting/__init__.py +++ b/src/plotting/__init__.py @@ -30,10 +30,10 @@ "projection": _PROJECTIONS["orthographic"], }, # The domains which are originally called "centraleurope" and "switzerland" - # are mostly the same. I suggest making domain "switzerland" much smaller, - # so that more spatial detail can be seen, especially in the complex - # topography of the alps. Also, make domain "centraleurope" slightly smaller, - # to zoom in more on the LAM area, so that less plotting area outside is lost. + # are mostly the same. I suggest making domain "switzerland" much smaller, + # so that more spatial detail can be seen, especially in the complex + # topography of the alps. Also, make domain "centraleurope" slightly smaller, + # to zoom in more on the LAM area, so that less plotting area outside is lost. "centraleurope": { "extent": [-1.5, 18, 41.5, 51], "projection": _PROJECTIONS["orthographic"], diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index adcd3877..4efa5b17 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -4,7 +4,6 @@ from matplotlib import pyplot as plt import warnings from .colormap_loader import load_ncl_colormap -from matplotlib.colors import BoundaryNorm import numpy as np @@ -112,47 +111,139 @@ def _fallback(): 120.0, ], }, - # hard-code this for the moment, can still make smarter later on: # RMSE and MAE first (is all the same). Sequential colour map to reflect the nature of the data (error, all positive). # Red is suggestive of 'bad' (high error). - # Use a limited number of levels so that absolute values of error can be read from the map. + # Use a limited number of levels so that absolute values of error can be read from the map. # always start at 0 so that the saturation of the colour corresponds to the error magnitude. - # RMSE: - "U_10M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "V_10M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "SP_10M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "TD_2M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "T_2M.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "PMSL.RMSE.map": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, - "PS.RMSE.map": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, - "TOT_PREC.RMSE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, - # would ideally want a 6th colour on the high end of the colour scale, but somehow + "U_10M.RMSE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "m/s"}, + "V_10M.RMSE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "m/s"}, + "SP_10M.RMSE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "m/s"}, + "TD_2M.RMSE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "°C"}, + "T_2M.RMSE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "°C"}, + "PMSL.RMSE.map": { + "cmap": plt.get_cmap("Reds", 7), + "levels": [0, 50, 100, 150, 200, 250, 300, 350], + } + | {"units": "Pa"}, + "PS.RMSE.map": { + "cmap": plt.get_cmap("Reds", 7), + "levels": [0, 50, 100, 150, 200, 250, 300, 350], + } + | {"units": "Pa"}, + "TOT_PREC.RMSE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 1, 1.5, 2, 3, 4], + } + | {"units": "mm"}, + # would ideally want a 6th colour on the high end of the colour scale, but somehow # matplotlib does not do that -> ? - # MAE: - "U_10M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "V_10M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "SP_10M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, - "TD_2M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "T_2M.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, - "PMSL.MAE.map": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, - "PS.MAE.map": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, - "TOT_PREC.MAE.map": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, - # the levels for precip are a bit on the bright side, but still worth keeping consistent with RMSE. - + "U_10M.MAE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "m/s"}, + "V_10M.MAE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "m/s"}, + "SP_10M.MAE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "m/s"}, + "TD_2M.MAE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "°C"}, + "T_2M.MAE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3], + } + | {"units": "°C"}, + "PMSL.MAE.map": { + "cmap": plt.get_cmap("Reds", 7), + "levels": [0, 50, 100, 150, 200, 250, 300, 350], + } + | {"units": "Pa"}, + "PS.MAE.map": { + "cmap": plt.get_cmap("Reds", 7), + "levels": [0, 50, 100, 150, 200, 250, 300, 350], + } + | {"units": "Pa"}, + "TOT_PREC.MAE.map": { + "cmap": plt.get_cmap("Reds", 6), + "levels": [0, 1, 1.5, 2, 3, 4], + } + | {"units": "mm"}, + # the levels for precip are a bit on the bright side, but still worth keeping consistent with RMSE. # Bias: # diverging colour scheme for the Bias to reflect the nature of the data (can be positive or negative, symmetric). # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. - "U_10M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, - "V_10M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, - "SP_10M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, - "TD_2M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, - "T_2M.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, - "PMSL.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, - "PS.BIAS.map": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, - "TOT_PREC.BIAS.map": {"cmap": plt.get_cmap("BrBG", 9), "levels": [-1, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 1]} | {"units": "mm"} + "U_10M.BIAS.map": { + "cmap": plt.get_cmap("RdBu_r", 9), + "levels": np.arange(start=-2.25, stop=2.26, step=0.5), + } + | {"units": "m/s"}, + "V_10M.BIAS.map": { + "cmap": plt.get_cmap("RdBu_r", 9), + "levels": np.arange(start=-2.25, stop=2.26, step=0.5), + } + | {"units": "m/s"}, + "SP_10M.BIAS.map": { + "cmap": plt.get_cmap("RdBu_r", 9), + "levels": np.arange(start=-2.25, stop=2.26, step=0.5), + } + | {"units": "m/s"}, + "TD_2M.BIAS.map": { + "cmap": plt.get_cmap("RdBu_r", 11), + "levels": np.arange(start=-2.75, stop=2.76, step=0.5), + } + | {"units": "°C"}, + "T_2M.BIAS.map": { + "cmap": plt.get_cmap("RdBu_r", 11), + "levels": np.arange(start=-2.75, stop=2.76, step=0.5), + } + | {"units": "°C"}, + "PMSL.BIAS.map": { + "cmap": plt.get_cmap("RdBu_r", 11), + "levels": np.arange(start=-110, stop=111, step=20), + } + | {"units": "Pa"}, + "PS.BIAS.map": { + "cmap": plt.get_cmap("RdBu_r", 11), + "levels": np.arange(start=-110, stop=111, step=20), + } + | {"units": "Pa"}, + "TOT_PREC.BIAS.map": { + "cmap": plt.get_cmap("BrBG", 9), + "levels": [-1, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 1], + } + | {"units": "mm"}, } CMAP_DEFAULTS = defaultdict(_fallback, _CMAP_DEFAULTS) diff --git a/workflow/Snakefile b/workflow/Snakefile index 51a25a4a..0516f1ef 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -141,21 +141,21 @@ rule metric_maps_all: expand( rules.plot_summary_stat_maps.output, run_id=collect_all_candidates(), - leadtime = config["metric_maps"]["leadtimes"], - metric = config["metric_maps"]["metrics"], - param = config["metric_maps"]["params"], - region = config["metric_maps"]["regions"], - season = config["metric_maps"]["seasons"], + leadtime=config["metric_maps"]["leadtimes"], + metric=config["metric_maps"]["metrics"], + param=config["metric_maps"]["params"], + region=config["metric_maps"]["regions"], + season=config["metric_maps"]["seasons"], experiment=EXPERIMENT_NAME, ), expand( rules.plot_summary_stat_maps_baseline.output, baseline_id=list(BASELINE_CONFIGS), - leadtime = config["metric_maps"]["leadtimes"], - metric = config["metric_maps"]["metrics"], - param = config["metric_maps"]["params"], - region = config["metric_maps"]["regions"], - season = config["metric_maps"]["seasons"], + leadtime=config["metric_maps"]["leadtimes"], + metric=config["metric_maps"]["metrics"], + param=config["metric_maps"]["params"], + region=config["metric_maps"]["regions"], + season=config["metric_maps"]["seasons"], experiment=EXPERIMENT_NAME, ), diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 28ae9022..54048ef3 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -150,16 +150,18 @@ rule plot_summary_stat_maps: script="workflow/scripts/plot_summary_stat_maps.mo.py", verif_file=OUT_ROOT / "data/runs/{run_id}/metric_maps/{param}_{leadtime}.nc", output: - OUT_ROOT / "results/{experiment}/metric_maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + OUT_ROOT + / "results/{experiment}/metric_maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits log: - OUT_ROOT / "logs/plot_summary_stat_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", + OUT_ROOT + / "logs/plot_summary_stat_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", resources: slurm_partition="postproc", cpus_per_task=1, runtime="10m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) @@ -174,11 +176,15 @@ rule plot_summary_stat_maps: # --season {wildcards.season} """ + use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: input: script="workflow/scripts/plot_summary_stat_maps.mo.py", - verif_file=OUT_ROOT / "data/baselines/{baseline_id}/metric_maps/{param}_{leadtime}.nc", + verif_file=OUT_ROOT + / "data/baselines/{baseline_id}/metric_maps/{param}_{leadtime}.nc", output: - OUT_ROOT / "results/{experiment}/metric_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + OUT_ROOT + / "results/{experiment}/metric_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", log: - OUT_ROOT / "logs/plot_summary_stat_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", \ No newline at end of file + OUT_ROOT + / "logs/plot_summary_stat_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 9f182ad3..bb96c6c9 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -34,7 +34,7 @@ rule verification_metrics_baseline: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} \ @@ -82,7 +82,7 @@ rule verification_metrics: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} \ @@ -120,7 +120,7 @@ rule verification_metrics_aggregation: cpus_per_task=24, mem_mb=250_000, runtime="2h", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} {input.verif_nc} --output {output} > {log} 2>&1 @@ -163,6 +163,7 @@ rule verification_metrics_plot: uv run {input.script} {input.verif} --output_dir {output} > {log} 2>&1 """ + rule verification_metrics_maps: input: "src/verification/__init__.py", @@ -190,7 +191,7 @@ rule verification_metrics_maps: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} \ @@ -202,6 +203,7 @@ rule verification_metrics_maps: --output {output} > {log} 2>&1 """ + rule verification_metrics_maps_baseline: input: script="workflow/scripts/verification_metric_maps.py", @@ -217,12 +219,13 @@ rule verification_metrics_maps_baseline: baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: - OUT_ROOT / "logs/verification_metrics_maps_baseline/{baseline_id}-{param}-{leadtime}.log", + OUT_ROOT + / "logs/verification_metrics_maps_baseline/{baseline_id}-{param}-{leadtime}.log", resources: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} \ diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py index c7f9e32f..dd2cd1f0 100644 --- a/workflow/scripts/plot_summary_stat_maps.mo.py +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -10,7 +10,6 @@ def _(): from argparse import ArgumentParser from pathlib import Path - import cartopy.crs as ccrs import earthkit.plots as ekp import numpy as np import xarray as xr @@ -18,6 +17,7 @@ def _(): from plotting import DOMAINS from plotting import StatePlotter from plotting.colormap_defaults import CMAP_DEFAULTS + return ( ArgumentParser, CMAP_DEFAULTS, @@ -44,15 +44,24 @@ def _(ArgumentParser, Path, np): parser = ArgumentParser() parser.add_argument( - "--input", type=str, default=None, help="Directory to .nc data containing the error fields" + "--input", + type=str, + default=None, + help="Directory to .nc data containing the error fields", ) parser.add_argument("--outfn", type=str, help="output filename") parser.add_argument("--leadtime", type=str, help="leadtime") parser.add_argument("--param", type=str, help="parameter") parser.add_argument("--region", type=str, help="name of region") - parser.add_argument("--metric", type=str, help = "Evaluation Metric. So far Bias, RMSE or MAE are implemented.") + parser.add_argument( + "--metric", + type=str, + help="Evaluation Metric. So far Bias, RMSE or MAE are implemented.", + ) parser.add_argument("--season", type=str, default="all", help="season filter") - parser.add_argument("--init_hour", type=str, default="all", help="initialization hour filter") + parser.add_argument( + "--init_hour", type=str, default="all", help="initialization hour filter" + ) args = parser.parse_args() verif_file = Path(args.input) @@ -70,7 +79,7 @@ def _(ArgumentParser, Path, np): else: raise ValueError("init_hour must be 'all' or an integer hour") - lead_time = np.timedelta64(lead_time, 'h') + lead_time = np.timedelta64(lead_time, "h") return ( init_hour, lead_time, @@ -90,10 +99,18 @@ def _(LOG, metric, param, season, verif_file, xr): var = f"{param}.{metric}" LOG.info("Selecting variable '%s' for season '%s'", var, season) ds = ds[var].sel(season=season) - LOG.info("Selected DataArray: dims=%s, shape=%s, dtype=%s", ds.dims, ds.shape, ds.dtype) - LOG.info("Value range: min=%.4g, max=%.4g, n_nan=%d", float(ds.min()), float(ds.max()), int(ds.isnull().sum())) + LOG.info( + "Selected DataArray: dims=%s, shape=%s, dtype=%s", ds.dims, ds.shape, ds.dtype + ) + LOG.info( + "Value range: min=%.4g, max=%.4g, n_nan=%d", + float(ds.min()), + float(ds.max()), + int(ds.isnull().sum()), + ) return (ds,) + @app.cell def _(CMAP_DEFAULTS, ekp): def get_style(param, metric, units_override=None): @@ -102,7 +119,11 @@ def get_style(param, metric, units_override=None): still needs to be passed as arguments to tripcolor()/tricontourf(). """ metric_key = f"{param}.{metric}.map" - cfg = CMAP_DEFAULTS[metric_key] if metric_key in CMAP_DEFAULTS else CMAP_DEFAULTS.get(param, {}) + cfg = ( + CMAP_DEFAULTS[metric_key] + if metric_key in CMAP_DEFAULTS + else CMAP_DEFAULTS.get(param, {}) + ) units = units_override if units_override is not None else cfg.get("units", "") return { "style": ekp.styles.Style( @@ -118,6 +139,7 @@ def get_style(param, metric, units_override=None): "vmax": cfg.get("vmax", None), "colors": cfg.get("colors", None), } + return (get_style,) @@ -137,7 +159,6 @@ def _( season, ): # plot individual fields - import matplotlib.pyplot as plt plotter = StatePlotter( ds["lon"].values.ravel(), @@ -160,8 +181,14 @@ def _( LOG.info("style_kwargs: %s", style_kwargs) if np.all(np.isnan(plot_vals)): - LOG.warning("All values are NaN for %s %s season=%s — plotting empty map.", param, metric, season) + LOG.warning( + "All values are NaN for %s %s season=%s — plotting empty map.", + param, + metric, + season, + ) import matplotlib.patches as mpatches + subplot.ax.set_facecolor("#cccccc") subplot.standard_layers() grey_patch = mpatches.Patch(color="#cccccc", label="No data") @@ -170,11 +197,10 @@ def _( plotter.plot_field(subplot, plot_vals, **style_kwargs) # black coast lines and country borders for better visibility - # grey is hardly visible, especially when the shading colours are intense. + # grey is hardly visible, especially when the shading colours are intense. subplot.coastlines(edgecolor="black", linewidth=1.0, zorder=5) subplot.borders(edgecolor="black", linewidth=0.5, zorder=5) - fig.title(f"{metric} of {param}, Season: {season}, Lead Time: {lead_time}") fig.save(outfn, bbox_inches="tight", dpi=200) diff --git a/workflow/scripts/verification_metric_maps.py b/workflow/scripts/verification_metric_maps.py index acee787f..411d53e5 100644 --- a/workflow/scripts/verification_metric_maps.py +++ b/workflow/scripts/verification_metric_maps.py @@ -48,6 +48,7 @@ def _season_of(dt: datetime) -> str: return "JJA" return "SON" + # Maps from standard parameter names to zarr variable names. # COSMO-2e zarrs use short CF names; COSMO-1e zarrs keep the COSMO names. _PARAMS_MAP_CO2 = { @@ -82,6 +83,7 @@ def _compute_derived(ds: xr.Dataset, param: str) -> xr.DataArray: # Truth loading # --------------------------------------------------------------------------- + def _open_zarr_component(root: Path, param: str) -> xr.DataArray: """Open a single native zarr variable lazily as a DataArray.""" zarr_param = _params_map(root)[param] @@ -111,11 +113,15 @@ def _open_zarr_component(root: Path, param: str) -> xr.DataArray: if lat is not None and lon is not None: if spatial_dim is not None: # flat 1-D case: cell/values dim - da = da.assign_coords(lat=(spatial_dim, lat.values), lon=(spatial_dim, lon.values)) + da = da.assign_coords( + lat=(spatial_dim, lat.values), lon=(spatial_dim, lon.values) + ) else: # 2-D case: lat/lon still on original flat index — attach via unstack - da = da.assign_coords(lat=(["y", "x"], lat.values.reshape(ny, nx)), - lon=(["y", "x"], lon.values.reshape(ny, nx))) + da = da.assign_coords( + lat=(["y", "x"], lat.values.reshape(ny, nx)), + lon=(["y", "x"], lon.values.reshape(ny, nx)), + ) return da @@ -162,6 +168,7 @@ def _preceding_step(grib_dir: Path, step: int) -> int | None: # Init-time discovery # --------------------------------------------------------------------------- + def iter_init_dirs(run_root: Path) -> list[tuple[datetime, Path]]: """Return ``(reftime, grib_dir)`` pairs for every complete init time. @@ -207,6 +214,7 @@ def iter_baseline_init_times(zarr_paths: list[Path], step: int) -> list[datetime # Main # --------------------------------------------------------------------------- + def main(args: Namespace) -> None: LOG.info("=" * 60) LOG.info("Spatial verification param=%s step=%dh", args.param, args.step) @@ -224,11 +232,16 @@ def main(args: Namespace) -> None: # Rename flat spatial dim to 'values' if the zarr uses 'cell'. if "cell" in truth_da.dims: truth_da = truth_da.rename({"cell": "values"}) - truth_times = set(truth_da.time.values) # keep as datetime64, tolist() yields ints for ns precision + truth_times = set( + truth_da.time.values + ) # keep as datetime64, tolist() yields ints for ns precision LOG.info("Truth opened lazily: %s", truth_da) if args.baseline_root: - init_items = [(rt, None) for rt in iter_baseline_init_times(args.baseline_zarrs, args.step)] + init_items = [ + (rt, None) + for rt in iter_baseline_init_times(args.baseline_zarrs, args.step) + ] LOG.info("Found %d baseline init times", len(init_items)) else: init_items = iter_init_dirs(args.run_root) @@ -248,8 +261,8 @@ def main(args: Namespace) -> None: # Running accumulators per season – initialised on the first successfully # processed sample so that we can infer the spatial shape from the data. # Each entry is a numpy array over the spatial dimension(s). - accum_n: dict[str, np.ndarray | None] = {s: None for s in SEASONS} - accum_sum_e: dict[str, np.ndarray | None] = {s: None for s in SEASONS} + accum_n: dict[str, np.ndarray | None] = {s: None for s in SEASONS} + accum_sum_e: dict[str, np.ndarray | None] = {s: None for s in SEASONS} accum_sum_se: dict[str, np.ndarray | None] = {s: None for s in SEASONS} accum_sum_ae: dict[str, np.ndarray | None] = {s: None for s in SEASONS} ref_truth_slice: xr.DataArray | None = None # kept for output coordinates @@ -274,7 +287,9 @@ def main(args: Namespace) -> None: first_iter = n_ok == 0 # --- load forecast --- - fct_params = list(_DERIVED[args.param]) if args.param in _DERIVED else [args.param] + fct_params = ( + list(_DERIVED[args.param]) if args.param in _DERIVED else [args.param] + ) try: if args.baseline_root: @@ -291,8 +306,12 @@ def main(args: Namespace) -> None: if args.param in _CUMULATIVE_PARAMS: prev_step = _preceding_step(grib_dir, args.step) if prev_step is None: - LOG.warning("No preceding step for cumulative param %s at step %d, skipping %s", - args.param, args.step, reftime) + LOG.warning( + "No preceding step for cumulative param %s at step %d, skipping %s", + args.param, + args.step, + reftime, + ) n_skip += 1 continue load_steps = [prev_step, args.step] @@ -322,26 +341,44 @@ def main(args: Namespace) -> None: fcst_raw = fcst[args.param].values if args.param in fcst else None if fcst_raw is not None: n_nan_fcst = int(np.isnan(fcst_raw).sum()) - LOG.info("fcst[%s]: shape=%s, min=%.4g, max=%.4g, n_nan=%d", - args.param, fcst_raw.shape, - float(np.nanmin(fcst_raw)) if n_nan_fcst < fcst_raw.size else float("nan"), - float(np.nanmax(fcst_raw)) if n_nan_fcst < fcst_raw.size else float("nan"), - n_nan_fcst) + LOG.info( + "fcst[%s]: shape=%s, min=%.4g, max=%.4g, n_nan=%d", + args.param, + fcst_raw.shape, + float(np.nanmin(fcst_raw)) + if n_nan_fcst < fcst_raw.size + else float("nan"), + float(np.nanmax(fcst_raw)) + if n_nan_fcst < fcst_raw.size + else float("nan"), + n_nan_fcst, + ) # --- load truth slice --- truth_slice = truth_da.sel(time=valid_time).compute() # For derived variables truth_da is already the derived DataArray, # so wrap it in a Dataset for map_forecast_to_truth compatibility. - truth_ds = truth_slice.to_dataset(name=args.param) if isinstance(truth_slice, xr.DataArray) else truth_slice + truth_ds = ( + truth_slice.to_dataset(name=args.param) + if isinstance(truth_slice, xr.DataArray) + else truth_slice + ) if first_iter: truth_raw = truth_slice.values n_nan_truth = int(np.isnan(truth_raw).sum()) - LOG.info("truth_slice[%s]: shape=%s, min=%.4g, max=%.4g, n_nan=%d", - args.param, truth_raw.shape, - float(np.nanmin(truth_raw)) if n_nan_truth < truth_raw.size else float("nan"), - float(np.nanmax(truth_raw)) if n_nan_truth < truth_raw.size else float("nan"), - n_nan_truth) + LOG.info( + "truth_slice[%s]: shape=%s, min=%.4g, max=%.4g, n_nan=%d", + args.param, + truth_raw.shape, + float(np.nanmin(truth_raw)) + if n_nan_truth < truth_raw.size + else float("nan"), + float(np.nanmax(truth_raw)) + if n_nan_truth < truth_raw.size + else float("nan"), + n_nan_truth, + ) # --- map forecast onto truth grid --- try: @@ -362,28 +399,41 @@ def main(args: Namespace) -> None: if first_iter: n_nan_mapped = int(np.isnan(fcst_vals).sum()) - LOG.info("fcst_mapped[%s]: shape=%s, min=%.4g, max=%.4g, n_nan=%d", - args.param, fcst_vals.shape, - float(np.nanmin(fcst_vals)) if n_nan_mapped < fcst_vals.size else float("nan"), - float(np.nanmax(fcst_vals)) if n_nan_mapped < fcst_vals.size else float("nan"), - n_nan_mapped) + LOG.info( + "fcst_mapped[%s]: shape=%s, min=%.4g, max=%.4g, n_nan=%d", + args.param, + fcst_vals.shape, + float(np.nanmin(fcst_vals)) + if n_nan_mapped < fcst_vals.size + else float("nan"), + float(np.nanmax(fcst_vals)) + if n_nan_mapped < fcst_vals.size + else float("nan"), + n_nan_mapped, + ) n_nan_err = int(np.isnan(error).sum()) - LOG.info("error: shape=%s, min=%.4g, max=%.4g, n_nan=%d / %d", - error.shape, - float(np.nanmin(error)) if n_nan_err < error.size else float("nan"), - float(np.nanmax(error)) if n_nan_err < error.size else float("nan"), - n_nan_err, error.size) + LOG.info( + "error: shape=%s, min=%.4g, max=%.4g, n_nan=%d / %d", + error.shape, + float(np.nanmin(error)) if n_nan_err < error.size else float("nan"), + float(np.nanmax(error)) if n_nan_err < error.size else float("nan"), + n_nan_err, + error.size, + ) n_nan_error = int(np.isnan(error).sum()) if n_nan_error == error.size: - LOG.warning("reftime=%s: error is all-NaN (%d points) — nothing accumulated.", - reftime.strftime(DATETIME_FMT), error.size) + LOG.warning( + "reftime=%s: error is all-NaN (%d points) — nothing accumulated.", + reftime.strftime(DATETIME_FMT), + error.size, + ) # --- initialise accumulators on first valid sample --- if accum_n["all"] is None: for s in SEASONS: - accum_n[s] = np.zeros(error.shape, dtype=np.int64) - accum_sum_e[s] = np.zeros(error.shape, dtype=np.float64) + accum_n[s] = np.zeros(error.shape, dtype=np.int64) + accum_sum_e[s] = np.zeros(error.shape, dtype=np.float64) accum_sum_se[s] = np.zeros(error.shape, dtype=np.float64) accum_sum_ae[s] = np.zeros(error.shape, dtype=np.float64) ref_truth_slice = truth_slice @@ -392,8 +442,8 @@ def main(args: Namespace) -> None: season = _season_of(reftime) valid = ~np.isnan(error) for s in [season, "all"]: - accum_n[s][valid] += 1 - accum_sum_e[s][valid] += error[valid] + accum_n[s][valid] += 1 + accum_sum_e[s][valid] += error[valid] accum_sum_se[s][valid] += error[valid] ** 2 accum_sum_ae[s][valid] += np.abs(error[valid]) n_ok += 1 @@ -435,9 +485,7 @@ def _seasonal_da(compute_fn) -> xr.DataArray: f"{args.param}.MAE": _seasonal_da( lambda n, s: np.where(n > 0, accum_sum_ae[s] / n, np.nan) ), - f"{args.param}.N": _seasonal_da( - lambda n, s: np.where(n > 0, n, np.nan) - ), + f"{args.param}.N": _seasonal_da(lambda n, s: np.where(n > 0, n, np.nan)), }, attrs={ "param": args.param, @@ -521,6 +569,8 @@ def _seasonal_da(compute_fn) -> xr.DataArray: if args.output is None: source = args.run_root or args.baseline_root - args.output = source / f"verification_metric_maps_{args.param}_step{args.step:03d}h.nc" + args.output = ( + source / f"verification_metric_maps_{args.param}_step{args.step:03d}h.nc" + ) main(args) diff --git a/workflow/scripts/verification_metrics.py b/workflow/scripts/verification_metrics.py index 6b64c85b..79bc71ae 100644 --- a/workflow/scripts/verification_metrics.py +++ b/workflow/scripts/verification_metrics.py @@ -75,7 +75,7 @@ def main(args: ScriptConfig): for ds in [fcst, truth]: if "U_10M" in ds and "V_10M" in ds: LOG.info("Calculating Wind Speed (SP_10M)...") - ds["SP_10M"] = (ds["U_10M"]**2 + ds["V_10M"]**2)**0.5 + ds["SP_10M"] = (ds["U_10M"] ** 2 + ds["V_10M"] ** 2) ** 0.5 # Optional: Add metadata for the netCDF output ds["SP_10M"].attrs = {"units": "m/s", "long_name": "10m Wind Speed"} From 94a7b0a73ba4413689b03950765750bf787e8052 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 17:42:49 +0200 Subject: [PATCH 093/129] Drop hardcoded SLURM node-exclude lines --- workflow/rules/plot.smk | 1 - workflow/rules/verification.smk | 5 ----- 2 files changed, 6 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 54048ef3..7c57d18f 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -161,7 +161,6 @@ rule plot_summary_stat_maps: slurm_partition="postproc", cpus_per_task=1, runtime="10m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index bb96c6c9..88078d64 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -34,7 +34,6 @@ rule verification_metrics_baseline: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} \ @@ -82,7 +81,6 @@ rule verification_metrics: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} \ @@ -120,7 +118,6 @@ rule verification_metrics_aggregation: cpus_per_task=24, mem_mb=250_000, runtime="2h", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} {input.verif_nc} --output {output} > {log} 2>&1 @@ -191,7 +188,6 @@ rule verification_metrics_maps: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} \ @@ -225,7 +221,6 @@ rule verification_metrics_maps_baseline: cpus_per_task=24, mem_mb=50_000, runtime="60m", - slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230", shell: """ uv run {input.script} \ From 413a1f645a3628d4c7d75605a3b9489bff286e42 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 5 May 2026 18:33:11 +0200 Subject: [PATCH 094/129] metric_maps config section for interpolator too. --- config/interpolators-ich1.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/config/interpolators-ich1.yaml b/config/interpolators-ich1.yaml index 5cb8e175..afc7bd09 100644 --- a/config/interpolators-ich1.yaml +++ b/config/interpolators-ich1.yaml @@ -71,3 +71,29 @@ profile: jobs: 50 batch_rules: plot_forecast_frame: 32 + +metric_maps: + params: + - T_2M + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC + leadtimes: + - 24 + metrics: + - BIAS + # - RMSE + # - MAE + regions: + - switzerland + # - centraleurope + seasons: + - all + # - DJF + # - MAM + # - JJA + # - SON From ececb1e18a20e9b6ad4b20766dc21bce6f26f193 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 10:57:26 +0200 Subject: [PATCH 095/129] Fix single-leadtime TOT_PREC verification in maps rules Both loaders assumed >=2 lead times when disaggregating TOT_PREC, so the maps rules (which pass a single step) crashed: load_baseline_from_zarr with "fmin which has no identity" on the empty .diff(), and load_fct_data_from_grib with "not all values found in index 'lead_time'" when anemoi-inference omits step 0. Push step-0 augmentation into the loaders: fetch step 0 alongside the requested steps for cumulative-from-start params, synthesize it as 0 if the GRIB lacks it, and drop it from the output. The maps script's _preceding_step shim is no longer needed and is removed. Production callers (regular verification, plot rules) all start at step 0, so their behavior is unchanged. --- src/data_input/__init__.py | 46 +++++++++++++++++--- workflow/scripts/verification_metric_maps.py | 38 ++-------------- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index ec27e86f..a4070c39 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -115,7 +115,21 @@ def load_fct_data_from_grib( # adds noticeable overhead to verif_metrics and the plot rules. files = sorted(root.glob(f"{reftime:%Y%m%d%H%M}*.grib")) fds = data_source.FileDataSource(datafiles=files) - ds = grib_decoder.load(fds, {"param": params, "step": steps}) + # For TOT_PREC (cumulative-from-start) we need step 0 to disaggregate to a + # 0->step period accumulation even when the caller asks for a single step. + # anemoi-inference may omit step 0 from the GRIB; tolerate that and + # synthesize lead_time=0, TOT_PREC=0 below (cumulative-from-start has + # nothing accumulated at the IC by definition). + needs_step_zero = "TOT_PREC" in params and 0 not in steps + fetch_steps = [0, *steps] if needs_step_zero else list(steps) + ds = grib_decoder.load(fds, {"param": params, "step": fetch_steps}) + # grib_decoder.load may silently drop steps that aren't on disk + # (anemoi-inference often omits step 0 even with cumulative-from-start + # accumulation). Detect that here so the TOT_PREC block can synthesize + # lead_time=0, TOT_PREC=0 below. + zero_lt = np.timedelta64(0, "h") + loaded_lead_times = next(iter(ds.values())).lead_time.values + step_zero_synthetic = needs_step_zero and zero_lt not in loaded_lead_times for var, da in ds.items(): if "z" in da.dims and da.sizes["z"] == 1: ds[var] = da.squeeze("z", drop=True) @@ -123,11 +137,17 @@ def load_fct_data_from_grib( ds[var] = da.rename({"z": da.attrs["vcoord_type"]}) ds = xr.merge([ds[p].rename(p) for p in ds], compat="no_conflicts") lead_times = np.array(steps, dtype="timedelta64[h]") - # Restrict to the requested lead times so that the TOT_PREC disaggregation - # below operates on the correct step interval even if the GRIB contains - # extra (e.g. hourly) steps beyond those requested — e.g. when consuming - # output from an interpolator emulator or a baseline with sub-step output. - ds = ds.sel(lead_time=lead_times) + fetch_lead_times = np.array(fetch_steps, dtype="timedelta64[h]") + # Restrict to the lead times we'll work with (fetch_lead_times = requested + # steps + step 0 if needed). This drops any extra (e.g. hourly) steps the + # GRIB may contain beyond what we asked for — e.g. when consuming output + # from an interpolator emulator or a baseline with sub-step output. + if step_zero_synthetic: + # Step 0 is missing from the GRIB; reindex inserts NaN at lead 0, + # which the xr.where below replaces with 0. + ds = ds.sel(lead_time=lead_times).reindex(lead_time=fetch_lead_times) + else: + ds = ds.sel(lead_time=fetch_lead_times) if "TOT_PREC" in ds.data_vars: ## Disaggregate TOT_PREC from cumulative-from-start (expected when the ## accumulate_from_start_of_forecast post-processor is enabled in @@ -168,6 +188,8 @@ def load_fct_data_from_grib( ## small float-noise negatives to zero (anything below -0.1 mm has ## already been caught by the check above). ds = ds.assign(TOT_PREC=diff.clip(min=0.0).reindex(lead_time=lead_times)) + # Drop the auxiliary step 0 from any non-TOT_PREC variables. + ds = ds.sel(lead_time=lead_times) # make sure time coordinate is available, and valid_time is not if "valid_time" in ds.coords: ds = ds.rename({"valid_time": "time"}) @@ -194,10 +216,18 @@ def load_baseline_from_zarr( {"forecast_reference_time": "ref_time", "step": "lead_time"} ).sortby("lead_time") lead_times = np.array(steps, dtype="timedelta64[h]") + # For TOT_PREC (cumulative-from-start) we need step 0 in the slice so that + # .diff() yields a 0->step period accumulation even when the caller + # requested a single step. The extra step is dropped at the final reindex. + zero_lt = np.timedelta64(0, "h") + if "TOT_PREC" in params and zero_lt not in lead_times: + fetch_lead_times = np.concatenate([[zero_lt], lead_times]) + else: + fetch_lead_times = lead_times # Restrict to the requested lead times up-front so that the TOT_PREC # disaggregation below operates on the correct step interval, and so that # all other variables avoid loading unused hourly steps from the zarr. - baseline = baseline[params].sel(ref_time=reftime, lead_time=lead_times) + baseline = baseline[params].sel(ref_time=reftime, lead_time=fetch_lead_times) if "TOT_PREC" in baseline.data_vars: if baseline.TOT_PREC.units == "kg m-2": baseline = baseline.assign(TOT_PREC=lambda x: x.TOT_PREC / 1000) @@ -224,6 +254,8 @@ def load_baseline_from_zarr( baseline = baseline.assign( TOT_PREC=diff.clip(min=0.0).reindex(lead_time=lead_times) ) + # Drop the auxiliary step 0 from any non-TOT_PREC variables. + baseline = baseline.sel(lead_time=lead_times) baseline = baseline.assign_coords(time=baseline.ref_time + baseline.lead_time) if "latitude" in baseline.coords and "longitude" in baseline: baseline = baseline.rename({"latitude": "lat", "longitude": "lon"}) diff --git a/workflow/scripts/verification_metric_maps.py b/workflow/scripts/verification_metric_maps.py index 411d53e5..15ee2b0b 100644 --- a/workflow/scripts/verification_metric_maps.py +++ b/workflow/scripts/verification_metric_maps.py @@ -147,23 +147,6 @@ def open_truth_zarr(root: Path, param: str) -> xr.DataArray: # GRIB step helpers # --------------------------------------------------------------------------- -# Parameters whose GRIB values are cumulative totals and must be disaggregated -# via diff before verification. For these, the preceding step must also be -# loaded so that load_fct_data_from_grib's diff produces a valid result. -_CUMULATIVE_PARAMS = {"TOT_PREC"} - - -def _preceding_step(grib_dir: Path, step: int) -> int | None: - """Return the largest available step smaller than *step* in *grib_dir*.""" - available = sorted( - int(f.stem.split("_")[-1]) - for f in grib_dir.glob("*.grib") - if f.stem.split("_")[-1].isdigit() - ) - smaller = [s for s in available if s < step] - return smaller[-1] if smaller else None - - # --------------------------------------------------------------------------- # Init-time discovery # --------------------------------------------------------------------------- @@ -301,26 +284,13 @@ def main(args: Namespace) -> None: params=fct_params, ) else: - # Cumulative params (e.g. TOT_PREC) are disaggregated via diff inside - # load_fct_data_from_grib, so we need to also load the preceding step. - if args.param in _CUMULATIVE_PARAMS: - prev_step = _preceding_step(grib_dir, args.step) - if prev_step is None: - LOG.warning( - "No preceding step for cumulative param %s at step %d, skipping %s", - args.param, - args.step, - reftime, - ) - n_skip += 1 - continue - load_steps = [prev_step, args.step] - else: - load_steps = [args.step] + # The loaders handle cumulative-from-start disaggregation + # internally (including fetching step 0 when needed for + # TOT_PREC), so a single-step request is sufficient here. fcst = load_fct_data_from_grib( root=grib_dir, reftime=reftime, - steps=load_steps, + steps=[args.step], params=fct_params, ) except Exception as exc: From 5ed413ba6749bbf40e25c29f6f64c57e5158b68b Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 11:41:33 +0200 Subject: [PATCH 096/129] Rename plot_summary_stat_maps[_baseline] -> plot_metric_maps[_baseline] Aligns the plot rules and script with the rest of the metric-maps naming convention. No logic change; only rule names, the script filename, and the log path are renamed. Output paths are unchanged. --- workflow/Snakefile | 4 ++-- workflow/rules/plot.smk | 12 ++++++------ ...ummary_stat_maps.mo.py => plot_metric_maps.mo.py} | 0 3 files changed, 8 insertions(+), 8 deletions(-) rename workflow/scripts/{plot_summary_stat_maps.mo.py => plot_metric_maps.mo.py} (100%) diff --git a/workflow/Snakefile b/workflow/Snakefile index 0516f1ef..5249d7c0 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -139,7 +139,7 @@ rule metric_maps_all: """Target rule for metric maps (opt-in via evalml experiment --maps).""" input: expand( - rules.plot_summary_stat_maps.output, + rules.plot_metric_maps.output, run_id=collect_all_candidates(), leadtime=config["metric_maps"]["leadtimes"], metric=config["metric_maps"]["metrics"], @@ -149,7 +149,7 @@ rule metric_maps_all: experiment=EXPERIMENT_NAME, ), expand( - rules.plot_summary_stat_maps_baseline.output, + rules.plot_metric_maps_baseline.output, baseline_id=list(BASELINE_CONFIGS), leadtime=config["metric_maps"]["leadtimes"], metric=config["metric_maps"]["metrics"], diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 7c57d18f..d0298c49 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -144,10 +144,10 @@ rule make_forecast_animation: """ -rule plot_summary_stat_maps: +rule plot_metric_maps: localrule: True input: - script="workflow/scripts/plot_summary_stat_maps.mo.py", + script="workflow/scripts/plot_metric_maps.mo.py", verif_file=OUT_ROOT / "data/runs/{run_id}/metric_maps/{param}_{leadtime}.nc", output: OUT_ROOT @@ -156,7 +156,7 @@ rule plot_summary_stat_maps: leadtime=r"\d+", # only digits log: OUT_ROOT - / "logs/plot_summary_stat_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", + / "logs/plot_metric_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", resources: slurm_partition="postproc", cpus_per_task=1, @@ -176,9 +176,9 @@ rule plot_summary_stat_maps: """ -use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: +use rule plot_metric_maps as plot_metric_maps_baseline with: input: - script="workflow/scripts/plot_summary_stat_maps.mo.py", + script="workflow/scripts/plot_metric_maps.mo.py", verif_file=OUT_ROOT / "data/baselines/{baseline_id}/metric_maps/{param}_{leadtime}.nc", output: @@ -186,4 +186,4 @@ use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: / "results/{experiment}/metric_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", log: OUT_ROOT - / "logs/plot_summary_stat_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", + / "logs/plot_metric_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_metric_maps.mo.py similarity index 100% rename from workflow/scripts/plot_summary_stat_maps.mo.py rename to workflow/scripts/plot_metric_maps.mo.py From 4326de5c35441cbb1ada1d07b44eb697a6833bfa Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 15:03:40 +0200 Subject: [PATCH 097/129] Revert .gitignore changes to match main --- .gitignore | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitignore b/.gitignore index 87783f79..50b2ff7d 100644 --- a/.gitignore +++ b/.gitignore @@ -55,8 +55,3 @@ uv.lock # evalml .evalml_snakemake_cmd.txt -config/forecasters-co2-mod-01.yaml -config/forecasters-co2-mod-01.yaml -config/forecasters-ich1_mod_01.yaml -config/forecasters-ich1_mod_02_1yr.yaml -config/forecasters-ich1-oper-small_adj_lfr_05_Stage_E_only_default_exp.yaml From de72f2a12e49c7759f055820c94bd8eaa036aa4c Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 15:05:17 +0200 Subject: [PATCH 098/129] Revert changes to .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 50b2ff7d..6fb3445f 100644 --- a/.gitignore +++ b/.gitignore @@ -54,4 +54,4 @@ uv.lock *credentials* # evalml -.evalml_snakemake_cmd.txt +.evalml_snakemake_cmd.txt \ No newline at end of file From 52e60bf19fcbbe351e34eae3092bdb4072387220 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 15:07:40 +0200 Subject: [PATCH 099/129] ... --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6fb3445f..50b2ff7d 100644 --- a/.gitignore +++ b/.gitignore @@ -54,4 +54,4 @@ uv.lock *credentials* # evalml -.evalml_snakemake_cmd.txt \ No newline at end of file +.evalml_snakemake_cmd.txt From 90427953b1bb31b460c4db7210e01c6fa0da368e Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 15:12:24 +0200 Subject: [PATCH 100/129] Comment out local map plotting. Keep commented line for potential future interactive map plotting (e.g., design stuff like nicer country-border polygons, or plotting every pixel). --- workflow/rules/plot.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index d0298c49..0b96a77c 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -145,7 +145,7 @@ rule make_forecast_animation: rule plot_metric_maps: - localrule: True + # localrule: True input: script="workflow/scripts/plot_metric_maps.mo.py", verif_file=OUT_ROOT / "data/runs/{run_id}/metric_maps/{param}_{leadtime}.nc", From 4961065b39e67920d2abd4625ea303f2e6812a76 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 15:40:10 +0200 Subject: [PATCH 101/129] Drop ad-hoc SP_10M derivation from verification_metrics.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The on-the-fly wind-speed computation was a half-baked feature outside the scope of MRB-650 (which is about metric maps). Comprehensive wind-speed support — covering point verification, dashboard, and proper config-driven derivation — belongs in a separate focused PR. Map plots of SP_10M continue to work because the metric-maps pipeline derives wind speed independently in verification_metric_maps.py. --- workflow/scripts/verification_metrics.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/workflow/scripts/verification_metrics.py b/workflow/scripts/verification_metrics.py index ecf01bac..20f0e1fc 100644 --- a/workflow/scripts/verification_metrics.py +++ b/workflow/scripts/verification_metrics.py @@ -72,14 +72,6 @@ def main(args: ScriptConfig): fcst = map_forecast_to_truth(fcst, truth) truth = truth.sel(time=fcst.time) - # before verifying, calculate wind speed: - for ds in [fcst, truth]: - if "U_10M" in ds and "V_10M" in ds: - LOG.info("Calculating Wind Speed (SP_10M)...") - ds["SP_10M"] = (ds["U_10M"] ** 2 + ds["V_10M"] ** 2) ** 0.5 - # Optional: Add metadata for the netCDF output - ds["SP_10M"].attrs = {"units": "m/s", "long_name": "10m Wind Speed"} - # compute metrics and statistics results = verify( fcst, From 858f872a323a7b222beb104fb9071b07fa05ba0b Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 15:53:30 +0200 Subject: [PATCH 102/129] Skip unnecessary comment. --- src/plotting/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/plotting/__init__.py b/src/plotting/__init__.py index 80d54b47..08c200ca 100644 --- a/src/plotting/__init__.py +++ b/src/plotting/__init__.py @@ -29,11 +29,6 @@ "extent": [-16.0, 25.0, 30.0, 65.0], "projection": _PROJECTIONS["orthographic"], }, - # The domains which are originally called "centraleurope" and "switzerland" - # are mostly the same. I suggest making domain "switzerland" much smaller, - # so that more spatial detail can be seen, especially in the complex - # topography of the alps. Also, make domain "centraleurope" slightly smaller, - # to zoom in more on the LAM area, so that less plotting area outside is lost. "centraleurope": { "extent": [-1.5, 18, 41.5, 51], "projection": _PROJECTIONS["orthographic"], From 1244b7c9ee5166b614c7ef48d0d3a864cf655083 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 16:00:37 +0200 Subject: [PATCH 103/129] Drop another unnecessary comment. --- src/plotting/colormap_defaults.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 4efa5b17..ff5277cc 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -112,8 +112,8 @@ def _fallback(): ], }, # hard-code this for the moment, can still make smarter later on: - # RMSE and MAE first (is all the same). Sequential colour map to reflect the nature of the data (error, all positive). - # Red is suggestive of 'bad' (high error). + # RMSE and MAE first (is all the same). Sequential colour map to reflect the nature of the data + # (error, all positive). Red is suggestive of 'bad' (high error). # Use a limited number of levels so that absolute values of error can be read from the map. # always start at 0 so that the saturation of the colour corresponds to the error magnitude. # RMSE: @@ -157,8 +157,6 @@ def _fallback(): "levels": [0, 1, 1.5, 2, 3, 4], } | {"units": "mm"}, - # would ideally want a 6th colour on the high end of the colour scale, but somehow - # matplotlib does not do that -> ? # MAE: "U_10M.MAE.map": { "cmap": plt.get_cmap("Reds", 6), From 080509ead27684270afbe409d453bb2c08f72fe3 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 8 May 2026 16:09:06 +0200 Subject: [PATCH 104/129] Tidy colormap_defaults.py: drop wishful-thinking comments The "hard-code for the moment, can still make smarter later on" line was the same anti-pattern as the recently-removed proposal comments. Replace it (plus the surrounding rationale block) with a tight impersonal description of the colour-scheme choice. --- src/plotting/colormap_defaults.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index ff5277cc..568646da 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -111,11 +111,9 @@ def _fallback(): 120.0, ], }, - # hard-code this for the moment, can still make smarter later on: - # RMSE and MAE first (is all the same). Sequential colour map to reflect the nature of the data - # (error, all positive). Red is suggestive of 'bad' (high error). - # Use a limited number of levels so that absolute values of error can be read from the map. - # always start at 0 so that the saturation of the colour corresponds to the error magnitude. + # Sequential Reds for RMSE and MAE: error is non-negative, larger ⇒ darker. + # Levels start at 0 so saturation maps directly to error magnitude; + # discrete levels make absolute values readable from the colour bar. # RMSE: "U_10M.RMSE.map": { "cmap": plt.get_cmap("Reds", 6), From c6fffd11b242b9072f5f8781d2b616f079d3516b Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 11 May 2026 20:36:43 +0200 Subject: [PATCH 105/129] Add init_hour stratification to metric maps verification_metric_maps.py now keys error accumulators by (season, init_hour) instead of season only, producing netcdfs with a new init_hour dimension (integer hour, -999 = "all"; matches the convention in verification_aggregation.py). plot_metric_maps.mo.py + plot.smk + Snakefile wire init_hour through as a wildcard so per-init-hour plots can be requested via the new metric_maps.init_hours config field (default ["all"], unchanged behaviour). MetricMapsConfig + config.schema.json + the eight example configs gain the init_hours scaffolding. This lets future per-init-hour analyses re-aggregate from the existing netcdfs without re-reading GRIBs. --- config/forecasters-co1e.yaml | 6 ++ config/forecasters-co2-disentangled.yaml | 6 ++ config/forecasters-co2.yaml | 6 ++ config/forecasters-ich1-oper-fixed.yaml | 6 ++ config/forecasters-ich1-oper.yaml | 6 ++ config/forecasters-ich1.yaml | 6 ++ config/interpolators-co2.yaml | 6 ++ config/interpolators-ich1.yaml | 6 ++ src/evalml/config.py | 7 ++ workflow/Snakefile | 2 + workflow/rules/plot.smk | 13 +-- workflow/scripts/plot_metric_maps.mo.py | 22 +++-- workflow/scripts/verification_metric_maps.py | 85 +++++++++++--------- workflow/tools/config.schema.json | 11 +++ 14 files changed, 139 insertions(+), 49 deletions(-) diff --git a/config/forecasters-co1e.yaml b/config/forecasters-co1e.yaml index 01ccb61b..4a0b2f48 100644 --- a/config/forecasters-co1e.yaml +++ b/config/forecasters-co1e.yaml @@ -98,3 +98,9 @@ metric_maps: # - MAM # - JJA # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" diff --git a/config/forecasters-co2-disentangled.yaml b/config/forecasters-co2-disentangled.yaml index 5e345ddb..ac1e6eda 100644 --- a/config/forecasters-co2-disentangled.yaml +++ b/config/forecasters-co2-disentangled.yaml @@ -117,3 +117,9 @@ metric_maps: # - MAM # - JJA # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" diff --git a/config/forecasters-co2.yaml b/config/forecasters-co2.yaml index 03234fca..508596c1 100644 --- a/config/forecasters-co2.yaml +++ b/config/forecasters-co2.yaml @@ -94,3 +94,9 @@ metric_maps: # - MAM # - JJA # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index 4aac22a7..20cf6f70 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -110,3 +110,9 @@ metric_maps: # - MAM # - JJA # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 1ded1ea9..7dcfc7dc 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -105,3 +105,9 @@ metric_maps: # - MAM # - JJA # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index 3c21649b..d22cfb4a 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -117,3 +117,9 @@ metric_maps: # - MAM # - JJA # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" diff --git a/config/interpolators-co2.yaml b/config/interpolators-co2.yaml index 15a00458..c7bf09fa 100644 --- a/config/interpolators-co2.yaml +++ b/config/interpolators-co2.yaml @@ -125,3 +125,9 @@ metric_maps: # - MAM # - JJA # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" diff --git a/config/interpolators-ich1.yaml b/config/interpolators-ich1.yaml index 8584288d..592abb87 100644 --- a/config/interpolators-ich1.yaml +++ b/config/interpolators-ich1.yaml @@ -114,3 +114,9 @@ metric_maps: # - MAM # - JJA # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" diff --git a/src/evalml/config.py b/src/evalml/config.py index f1709e9a..a4bd71e2 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -234,6 +234,13 @@ class MetricMapsConfig(BaseModel): default=["all", "DJF", "MAM", "JJA", "SON"], description="List of seasons to plot.", ) + init_hours: List[str] = Field( + default=["all"], + description=( + "List of initialization hours to plot. Use 'all' for the unstratified " + "view, or zero-padded hour strings like '00', '06', '12', '18'." + ), + ) class Locations(BaseModel): diff --git a/workflow/Snakefile b/workflow/Snakefile index 5249d7c0..3a4b8c6e 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -146,6 +146,7 @@ rule metric_maps_all: param=config["metric_maps"]["params"], region=config["metric_maps"]["regions"], season=config["metric_maps"]["seasons"], + init_hour=config["metric_maps"]["init_hours"], experiment=EXPERIMENT_NAME, ), expand( @@ -156,6 +157,7 @@ rule metric_maps_all: param=config["metric_maps"]["params"], region=config["metric_maps"]["regions"], season=config["metric_maps"]["seasons"], + init_hour=config["metric_maps"]["init_hours"], experiment=EXPERIMENT_NAME, ), diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 0b96a77c..01672112 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -151,12 +151,13 @@ rule plot_metric_maps: verif_file=OUT_ROOT / "data/runs/{run_id}/metric_maps/{param}_{leadtime}.nc", output: OUT_ROOT - / "results/{experiment}/metric_maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + / "results/{experiment}/metric_maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{init_hour}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits + init_hour=r"all|\d{1,2}", log: OUT_ROOT - / "logs/plot_metric_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", + / "logs/plot_metric_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{init_hour}-{leadtime}.log", resources: slurm_partition="postproc", cpus_per_task=1, @@ -167,12 +168,12 @@ rule plot_metric_maps: uv run python {input.script} \ --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ - --season {wildcards.season} > {log} 2>&1 + --season {wildcards.season} --init_hour {wildcards.init_hour} > {log} 2>&1 # interactive editing (needs to set localrule: True and use only one core) # marimo edit {input.script} -- \ # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ - # --season {wildcards.season} + # --season {wildcards.season} --init_hour {wildcards.init_hour} """ @@ -183,7 +184,7 @@ use rule plot_metric_maps as plot_metric_maps_baseline with: / "data/baselines/{baseline_id}/metric_maps/{param}_{leadtime}.nc", output: OUT_ROOT - / "results/{experiment}/metric_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + / "results/{experiment}/metric_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{init_hour}_{leadtime}.png", log: OUT_ROOT - / "logs/plot_metric_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{leadtime}.log", + / "logs/plot_metric_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{init_hour}-{leadtime}.log", diff --git a/workflow/scripts/plot_metric_maps.mo.py b/workflow/scripts/plot_metric_maps.mo.py index dd2cd1f0..a718217a 100644 --- a/workflow/scripts/plot_metric_maps.mo.py +++ b/workflow/scripts/plot_metric_maps.mo.py @@ -77,7 +77,12 @@ def _(ArgumentParser, Path, np): if init_hour == "all": init_hour = -999 else: - raise ValueError("init_hour must be 'all' or an integer hour") + try: + init_hour = int(init_hour) + except ValueError as exc: + raise ValueError( + "init_hour must be 'all' or an integer hour" + ) from exc lead_time = np.timedelta64(lead_time, "h") return ( @@ -93,12 +98,14 @@ def _(ArgumentParser, Path, np): @app.cell -def _(LOG, metric, param, season, verif_file, xr): +def _(LOG, init_hour, metric, param, season, verif_file, xr): ds = xr.open_dataset(verif_file) LOG.info("Opened dataset: %s", ds) var = f"{param}.{metric}" - LOG.info("Selecting variable '%s' for season '%s'", var, season) - ds = ds[var].sel(season=season) + LOG.info( + "Selecting variable '%s' for season '%s', init_hour=%s", var, season, init_hour + ) + ds = ds[var].sel(season=season, init_hour=init_hour) LOG.info( "Selected DataArray: dims=%s, shape=%s, dtype=%s", ds.dims, ds.shape, ds.dtype ) @@ -150,6 +157,7 @@ def _( StatePlotter, ds, get_style, + init_hour, lead_time, metric, np, @@ -201,7 +209,11 @@ def _( subplot.coastlines(edgecolor="black", linewidth=1.0, zorder=5) subplot.borders(edgecolor="black", linewidth=0.5, zorder=5) - fig.title(f"{metric} of {param}, Season: {season}, Lead Time: {lead_time}") + init_hour_lbl = "all" if init_hour == -999 else f"{init_hour:02d}" + fig.title( + f"{metric} of {param}, Season: {season}, " + f"Init hour: {init_hour_lbl}, Lead Time: {lead_time}" + ) fig.save(outfn, bbox_inches="tight", dpi=200) LOG.info(f"saved: {outfn}") diff --git a/workflow/scripts/verification_metric_maps.py b/workflow/scripts/verification_metric_maps.py index 15ee2b0b..51002129 100644 --- a/workflow/scripts/verification_metric_maps.py +++ b/workflow/scripts/verification_metric_maps.py @@ -35,6 +35,8 @@ DATETIME_FMT = "%Y%m%d%H%M" SEASONS = ["DJF", "MAM", "JJA", "SON", "all"] +# Init hour buckets. -999 is the "all" sentinel (matches verification_aggregation.py). +INIT_HOURS = [0, 6, 12, 18, -999] def _season_of(dt: datetime) -> str: @@ -241,13 +243,15 @@ def main(args: Namespace) -> None: step_td = timedelta(hours=args.step) - # Running accumulators per season – initialised on the first successfully - # processed sample so that we can infer the spatial shape from the data. - # Each entry is a numpy array over the spatial dimension(s). - accum_n: dict[str, np.ndarray | None] = {s: None for s in SEASONS} - accum_sum_e: dict[str, np.ndarray | None] = {s: None for s in SEASONS} - accum_sum_se: dict[str, np.ndarray | None] = {s: None for s in SEASONS} - accum_sum_ae: dict[str, np.ndarray | None] = {s: None for s in SEASONS} + # Running accumulators keyed by (season, init_hour) – initialised on the + # first successfully processed sample so that we can infer the spatial + # shape from the data. Each entry is a numpy array over the spatial + # dimension(s). + bucket_keys = [(s, h) for s in SEASONS for h in INIT_HOURS] + accum_n: dict[tuple[str, int], np.ndarray | None] = {k: None for k in bucket_keys} + accum_sum_e: dict[tuple[str, int], np.ndarray | None] = {k: None for k in bucket_keys} + accum_sum_se: dict[tuple[str, int], np.ndarray | None] = {k: None for k in bucket_keys} + accum_sum_ae: dict[tuple[str, int], np.ndarray | None] = {k: None for k in bucket_keys} ref_truth_slice: xr.DataArray | None = None # kept for output coordinates n_ok = 0 @@ -400,22 +404,25 @@ def main(args: Namespace) -> None: ) # --- initialise accumulators on first valid sample --- - if accum_n["all"] is None: - for s in SEASONS: - accum_n[s] = np.zeros(error.shape, dtype=np.int64) - accum_sum_e[s] = np.zeros(error.shape, dtype=np.float64) - accum_sum_se[s] = np.zeros(error.shape, dtype=np.float64) - accum_sum_ae[s] = np.zeros(error.shape, dtype=np.float64) + if accum_n[("all", -999)] is None: + for k in bucket_keys: + accum_n[k] = np.zeros(error.shape, dtype=np.int64) + accum_sum_e[k] = np.zeros(error.shape, dtype=np.float64) + accum_sum_se[k] = np.zeros(error.shape, dtype=np.float64) + accum_sum_ae[k] = np.zeros(error.shape, dtype=np.float64) ref_truth_slice = truth_slice - # --- accumulate into the matching season bucket and "all" (NaN-safe) --- + # --- accumulate into matching (season, init_hour) buckets, plus the + # "all" rows/cols on each axis (NaN-safe) --- season = _season_of(reftime) + ih = reftime.hour valid = ~np.isnan(error) - for s in [season, "all"]: - accum_n[s][valid] += 1 - accum_sum_e[s][valid] += error[valid] - accum_sum_se[s][valid] += error[valid] ** 2 - accum_sum_ae[s][valid] += np.abs(error[valid]) + for s in (season, "all"): + for h in (ih, -999): + accum_n[(s, h)][valid] += 1 + accum_sum_e[(s, h)][valid] += error[valid] + accum_sum_se[(s, h)][valid] += error[valid] ** 2 + accum_sum_ae[(s, h)][valid] += np.abs(error[valid]) n_ok += 1 LOG.info("Finished: %d init times processed, %d skipped", n_ok, n_skip) @@ -424,7 +431,7 @@ def main(args: Namespace) -> None: LOG.error("No data could be processed – no output written.") return - # --- compute aggregate maps per season, then stack into a season dimension --- + # --- compute aggregate maps per (season, init_hour), then stack --- spatial_coords = { c: ref_truth_slice[c] for c in ref_truth_slice.coords @@ -432,30 +439,32 @@ def main(args: Namespace) -> None: and c != "time" } spatial_dims = list(ref_truth_slice.dims) - out_dims = ["season"] + spatial_dims - out_coords = {"season": SEASONS, **spatial_coords} - - def _seasonal_da(compute_fn) -> xr.DataArray: - """Stack per-season arrays into a (season, *spatial) DataArray.""" - slices = [] - for s in SEASONS: - n = accum_n[s] - with np.errstate(invalid="ignore", divide="ignore"): - slices.append(compute_fn(n, s).astype(np.float32)) - return xr.DataArray(np.stack(slices), dims=out_dims, coords=out_coords) + out_dims = ["season", "init_hour"] + spatial_dims + out_coords = {"season": SEASONS, "init_hour": INIT_HOURS, **spatial_coords} + + def _strat_da(compute_fn) -> xr.DataArray: + """Stack per-(season, init_hour) arrays into a (season, init_hour, *spatial) DataArray.""" + out_shape = (len(SEASONS), len(INIT_HOURS)) + ref_truth_slice.shape + arr = np.empty(out_shape, dtype=np.float32) + for i, s in enumerate(SEASONS): + for j, h in enumerate(INIT_HOURS): + n = accum_n[(s, h)] + with np.errstate(invalid="ignore", divide="ignore"): + arr[i, j] = compute_fn(n, s, h).astype(np.float32) + return xr.DataArray(arr, dims=out_dims, coords=out_coords) out = xr.Dataset( { - f"{args.param}.BIAS": _seasonal_da( - lambda n, s: np.where(n > 0, accum_sum_e[s] / n, np.nan) + f"{args.param}.BIAS": _strat_da( + lambda n, s, h: np.where(n > 0, accum_sum_e[(s, h)] / n, np.nan) ), - f"{args.param}.RMSE": _seasonal_da( - lambda n, s: np.where(n > 0, np.sqrt(accum_sum_se[s] / n), np.nan) + f"{args.param}.RMSE": _strat_da( + lambda n, s, h: np.where(n > 0, np.sqrt(accum_sum_se[(s, h)] / n), np.nan) ), - f"{args.param}.MAE": _seasonal_da( - lambda n, s: np.where(n > 0, accum_sum_ae[s] / n, np.nan) + f"{args.param}.MAE": _strat_da( + lambda n, s, h: np.where(n > 0, accum_sum_ae[(s, h)] / n, np.nan) ), - f"{args.param}.N": _seasonal_da(lambda n, s: np.where(n > 0, n, np.nan)), + f"{args.param}.N": _strat_da(lambda n, s, h: np.where(n > 0, n, np.nan)), }, attrs={ "param": args.param, diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index 149a2d01..2b57b96b 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -535,6 +535,17 @@ }, "title": "Seasons", "type": "array" + }, + "init_hours": { + "default": [ + "all" + ], + "description": "List of initialization hours to plot. Use 'all' for the unstratified view, or zero-padded hour strings like '00', '06', '12', '18'.", + "items": { + "type": "string" + }, + "title": "Init Hours", + "type": "array" } }, "title": "MetricMapsConfig", From 90ff476ee659a1cc1ff890989861ef181d02acfd Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 11 May 2026 20:40:53 +0200 Subject: [PATCH 106/129] Apply ruff-format to metric_maps scripts --- workflow/scripts/plot_metric_maps.mo.py | 4 +--- workflow/scripts/verification_metric_maps.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/workflow/scripts/plot_metric_maps.mo.py b/workflow/scripts/plot_metric_maps.mo.py index a718217a..2a41cd9e 100644 --- a/workflow/scripts/plot_metric_maps.mo.py +++ b/workflow/scripts/plot_metric_maps.mo.py @@ -80,9 +80,7 @@ def _(ArgumentParser, Path, np): try: init_hour = int(init_hour) except ValueError as exc: - raise ValueError( - "init_hour must be 'all' or an integer hour" - ) from exc + raise ValueError("init_hour must be 'all' or an integer hour") from exc lead_time = np.timedelta64(lead_time, "h") return ( diff --git a/workflow/scripts/verification_metric_maps.py b/workflow/scripts/verification_metric_maps.py index 51002129..a5e0d9cd 100644 --- a/workflow/scripts/verification_metric_maps.py +++ b/workflow/scripts/verification_metric_maps.py @@ -249,9 +249,15 @@ def main(args: Namespace) -> None: # dimension(s). bucket_keys = [(s, h) for s in SEASONS for h in INIT_HOURS] accum_n: dict[tuple[str, int], np.ndarray | None] = {k: None for k in bucket_keys} - accum_sum_e: dict[tuple[str, int], np.ndarray | None] = {k: None for k in bucket_keys} - accum_sum_se: dict[tuple[str, int], np.ndarray | None] = {k: None for k in bucket_keys} - accum_sum_ae: dict[tuple[str, int], np.ndarray | None] = {k: None for k in bucket_keys} + accum_sum_e: dict[tuple[str, int], np.ndarray | None] = { + k: None for k in bucket_keys + } + accum_sum_se: dict[tuple[str, int], np.ndarray | None] = { + k: None for k in bucket_keys + } + accum_sum_ae: dict[tuple[str, int], np.ndarray | None] = { + k: None for k in bucket_keys + } ref_truth_slice: xr.DataArray | None = None # kept for output coordinates n_ok = 0 @@ -459,7 +465,9 @@ def _strat_da(compute_fn) -> xr.DataArray: lambda n, s, h: np.where(n > 0, accum_sum_e[(s, h)] / n, np.nan) ), f"{args.param}.RMSE": _strat_da( - lambda n, s, h: np.where(n > 0, np.sqrt(accum_sum_se[(s, h)] / n), np.nan) + lambda n, s, h: np.where( + n > 0, np.sqrt(accum_sum_se[(s, h)] / n), np.nan + ) ), f"{args.param}.MAE": _strat_da( lambda n, s, h: np.where(n > 0, accum_sum_ae[(s, h)] / n, np.nan) From e9fbef94a100e805f1e723cc4f6aab8695bd9fab Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 1 Jun 2026 15:55:56 +0200 Subject: [PATCH 107/129] =?UTF-8?q?Rename=20"metric=20maps"=20=E2=86=92=20?= =?UTF-8?q?"score=20maps"=20for=20naming=20consistency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback: BIAS/RMSE/MAE compare forecast to truth, so by evalml's convention they're scores (not metrics, which are statistics of the forecast alone). Aligns the new pipeline's naming with the existing scores/metrics distinction throughout the codebase. - Config section: metric_maps -> score_maps - Pydantic class: MetricMapsConfig -> ScoreMapsConfig - Snakefile target: metric_maps_all -> score_maps_all - Rules: verification_metrics_maps[_baseline] -> verification_score_maps[_baseline] - Plot rules: plot_metric_maps[_baseline] -> plot_score_maps[_baseline] - Scripts: verification_metric_maps.py, plot_metric_maps.mo.py -> ..._score_maps... - Output / log directories: metric_maps/ -> score_maps/ CLI flag (--maps) and colormap-key suffix (.map) kept as-is — they're already concise and unambiguous. The existing main-side rules (verification_metrics, verification_metrics_aggregation, verification_metrics_plot, etc.) also compute scores against truth but are named "metrics". So main itself doesn't follow the reviewer's stated convention. That's not in MRB-650's scope to fix — but worth noting in case the reviewer wants to address it broadly. --- config/forecasters-co1e.yaml | 2 +- config/forecasters-co2-disentangled.yaml | 2 +- config/forecasters-co2.yaml | 2 +- config/forecasters-ich1-oper-fixed.yaml | 2 +- config/forecasters-ich1-oper.yaml | 2 +- config/forecasters-ich1.yaml | 2 +- config/interpolators-co2.yaml | 2 +- config/interpolators-ich1.yaml | 2 +- src/data_input/__init__.py | 2 +- src/evalml/cli.py | 4 +- src/evalml/config.py | 10 +-- workflow/Snakefile | 32 +++---- workflow/rules/plot.smk | 20 ++--- workflow/rules/verification.smk | 16 ++-- ...etric_maps.mo.py => plot_score_maps.mo.py} | 0 ...ric_maps.py => verification_score_maps.py} | 4 +- workflow/tools/config.schema.json | 88 +++++++++---------- 17 files changed, 96 insertions(+), 96 deletions(-) rename workflow/scripts/{plot_metric_maps.mo.py => plot_score_maps.mo.py} (100%) rename workflow/scripts/{verification_metric_maps.py => verification_score_maps.py} (99%) diff --git a/config/forecasters-co1e.yaml b/config/forecasters-co1e.yaml index 4a0b2f48..cf819cee 100644 --- a/config/forecasters-co1e.yaml +++ b/config/forecasters-co1e.yaml @@ -73,7 +73,7 @@ profile: batch_rules: plot_forecast_frame: 32 -metric_maps: +score_maps: params: - T_2M # - TD_2M diff --git a/config/forecasters-co2-disentangled.yaml b/config/forecasters-co2-disentangled.yaml index ac1e6eda..f5b27b2e 100644 --- a/config/forecasters-co2-disentangled.yaml +++ b/config/forecasters-co2-disentangled.yaml @@ -92,7 +92,7 @@ profile: gpus: 0 jobs: 50 -metric_maps: +score_maps: params: - T_2M # - TD_2M diff --git a/config/forecasters-co2.yaml b/config/forecasters-co2.yaml index 508596c1..6dc0b6c7 100644 --- a/config/forecasters-co2.yaml +++ b/config/forecasters-co2.yaml @@ -69,7 +69,7 @@ profile: batch_rules: plot_forecast_frame: 32 -metric_maps: +score_maps: params: - T_2M # - TD_2M diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index 20cf6f70..c9c4b419 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -85,7 +85,7 @@ profile: batch_rules: plot_forecast_frame: 32 -metric_maps: +score_maps: params: - T_2M # - TD_2M diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 7dcfc7dc..a7a19710 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -80,7 +80,7 @@ profile: batch_rules: plot_forecast_frame: 32 -metric_maps: +score_maps: params: - T_2M # - TD_2M diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index d22cfb4a..e27b5137 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -92,7 +92,7 @@ profile: batch_rules: plot_forecast_frame: 32 -metric_maps: +score_maps: params: - T_2M # - TD_2M diff --git a/config/interpolators-co2.yaml b/config/interpolators-co2.yaml index c7bf09fa..a718aea9 100644 --- a/config/interpolators-co2.yaml +++ b/config/interpolators-co2.yaml @@ -100,7 +100,7 @@ profile: batch_rules: plot_forecast_frame: 32 -metric_maps: +score_maps: params: - T_2M # - TD_2M diff --git a/config/interpolators-ich1.yaml b/config/interpolators-ich1.yaml index 592abb87..65c63b70 100644 --- a/config/interpolators-ich1.yaml +++ b/config/interpolators-ich1.yaml @@ -89,7 +89,7 @@ profile: batch_rules: plot_forecast_frame: 32 -metric_maps: +score_maps: params: - T_2M # - TD_2M diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index 8a6f0b76..71e49bf6 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -114,7 +114,7 @@ def load_fct_data_from_grib( # TODO: this function carries a large per-call setup cost that is # independent of data volume (likely eccodes/FileDataSource init or GRIB # index build, not decoding). It dominates runtime in any rule that calls - # it inside a per-init-time loop (e.g. verification_metrics_maps) and also + # it inside a per-init-time loop (e.g. verification_score_maps) and also # adds noticeable overhead to verif_metrics and the plot rules. files = sorted(root.glob(f"{reftime:%Y%m%d%H%M}*.grib")) fds = data_source.FileDataSource(datafiles=files) diff --git a/src/evalml/cli.py b/src/evalml/cli.py index 1c0fdfc7..fcc295ff 100644 --- a/src/evalml/cli.py +++ b/src/evalml/cli.py @@ -168,7 +168,7 @@ def cli(): "--maps", is_flag=True, default=False, - help="Also produce metric maps (computationally intensive).", + help="Also produce score maps (computationally intensive).", ) @workflow_options def experiment( @@ -194,7 +194,7 @@ def experiment( dag, rulegraph, extra_smk_args, - extra_targets=["metric_maps_all"] if maps else [], + extra_targets=["score_maps_all"] if maps else [], ) diff --git a/src/evalml/config.py b/src/evalml/config.py index a4bd71e2..82058452 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -208,8 +208,8 @@ class BaselineItem(BaseModel): baseline: BaselineConfig -class MetricMapsConfig(BaseModel): - """Parameters controlling which metric map plots are produced.""" +class ScoreMapsConfig(BaseModel): + """Parameters controlling which score map plots are produced.""" params: List[str] = Field( default=["T_2M", "TD_2M", "U_10M", "V_10M", "SP_10M", "PS", "PMSL", "TOT_PREC"], @@ -386,9 +386,9 @@ def validate_threshold_operators( dashboard: Dashboard locations: Locations profile: Profile - metric_maps: MetricMapsConfig = Field( - default_factory=MetricMapsConfig, - description="Parameters for metric map plots (used with --maps flag).", + score_maps: ScoreMapsConfig = Field( + default_factory=ScoreMapsConfig, + description="Parameters for score map plots (used with --maps flag).", ) model_config = { diff --git a/workflow/Snakefile b/workflow/Snakefile index 3a4b8c6e..a542af4a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -135,29 +135,29 @@ rule experiment_all: ), -rule metric_maps_all: - """Target rule for metric maps (opt-in via evalml experiment --maps).""" +rule score_maps_all: + """Target rule for score maps (opt-in via evalml experiment --maps).""" input: expand( - rules.plot_metric_maps.output, + rules.plot_score_maps.output, run_id=collect_all_candidates(), - leadtime=config["metric_maps"]["leadtimes"], - metric=config["metric_maps"]["metrics"], - param=config["metric_maps"]["params"], - region=config["metric_maps"]["regions"], - season=config["metric_maps"]["seasons"], - init_hour=config["metric_maps"]["init_hours"], + leadtime=config["score_maps"]["leadtimes"], + metric=config["score_maps"]["metrics"], + param=config["score_maps"]["params"], + region=config["score_maps"]["regions"], + season=config["score_maps"]["seasons"], + init_hour=config["score_maps"]["init_hours"], experiment=EXPERIMENT_NAME, ), expand( - rules.plot_metric_maps_baseline.output, + rules.plot_score_maps_baseline.output, baseline_id=list(BASELINE_CONFIGS), - leadtime=config["metric_maps"]["leadtimes"], - metric=config["metric_maps"]["metrics"], - param=config["metric_maps"]["params"], - region=config["metric_maps"]["regions"], - season=config["metric_maps"]["seasons"], - init_hour=config["metric_maps"]["init_hours"], + leadtime=config["score_maps"]["leadtimes"], + metric=config["score_maps"]["metrics"], + param=config["score_maps"]["params"], + region=config["score_maps"]["regions"], + season=config["score_maps"]["seasons"], + init_hour=config["score_maps"]["init_hours"], experiment=EXPERIMENT_NAME, ), diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 01672112..156c3a21 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -144,20 +144,20 @@ rule make_forecast_animation: """ -rule plot_metric_maps: +rule plot_score_maps: # localrule: True input: - script="workflow/scripts/plot_metric_maps.mo.py", - verif_file=OUT_ROOT / "data/runs/{run_id}/metric_maps/{param}_{leadtime}.nc", + script="workflow/scripts/plot_score_maps.mo.py", + verif_file=OUT_ROOT / "data/runs/{run_id}/score_maps/{param}_{leadtime}.nc", output: OUT_ROOT - / "results/{experiment}/metric_maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{init_hour}_{leadtime}.png", + / "results/{experiment}/score_maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{init_hour}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits init_hour=r"all|\d{1,2}", log: OUT_ROOT - / "logs/plot_metric_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{init_hour}-{leadtime}.log", + / "logs/plot_score_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{init_hour}-{leadtime}.log", resources: slurm_partition="postproc", cpus_per_task=1, @@ -177,14 +177,14 @@ rule plot_metric_maps: """ -use rule plot_metric_maps as plot_metric_maps_baseline with: +use rule plot_score_maps as plot_score_maps_baseline with: input: - script="workflow/scripts/plot_metric_maps.mo.py", + script="workflow/scripts/plot_score_maps.mo.py", verif_file=OUT_ROOT - / "data/baselines/{baseline_id}/metric_maps/{param}_{leadtime}.nc", + / "data/baselines/{baseline_id}/score_maps/{param}_{leadtime}.nc", output: OUT_ROOT - / "results/{experiment}/metric_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{init_hour}_{leadtime}.png", + / "results/{experiment}/score_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{init_hour}_{leadtime}.png", log: OUT_ROOT - / "logs/plot_metric_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{init_hour}-{leadtime}.log", + / "logs/plot_score_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{init_hour}-{leadtime}.log", diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 16782dc0..80eca8b5 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -166,11 +166,11 @@ rule verification_metrics_plot: """ -rule verification_metrics_maps: +rule verification_score_maps: input: "src/verification/__init__.py", "src/data_input/__init__.py", - script="workflow/scripts/verification_metric_maps.py", + script="workflow/scripts/verification_score_maps.py", inference_okfiles=lambda wc: expand( rules.inference_execute.output.okfile, init_time=_restrict_reftimes_to_hours(REFTIMES), @@ -178,7 +178,7 @@ rule verification_metrics_maps: ), truth=config["truth"]["root"], output: - OUT_ROOT / "data/runs/{run_id}/metric_maps/{param}_{leadtime}.nc", + OUT_ROOT / "data/runs/{run_id}/score_maps/{param}_{leadtime}.nc", # wildcard_constraints: # run_id="^" # to avoid ambiguitiy with run_baseline_verif # TODO: implement logic to use experiment name instead of run_id as wildcard @@ -188,7 +188,7 @@ rule verification_metrics_maps: truth_label=config["truth"]["label"], reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: - OUT_ROOT / "logs/verification_metrics_maps/{run_id}-{param}-{leadtime}.log", + OUT_ROOT / "logs/verification_score_maps/{run_id}-{param}-{leadtime}.log", resources: cpus_per_task=24, mem_mb=50_000, @@ -205,9 +205,9 @@ rule verification_metrics_maps: """ -rule verification_metrics_maps_baseline: +rule verification_score_maps_baseline: input: - script="workflow/scripts/verification_metric_maps.py", + script="workflow/scripts/verification_score_maps.py", baseline_zarrs=lambda wc: expand( "{root}/FCST{year}.zarr", root=BASELINE_CONFIGS[wc.baseline_id].get("root"), @@ -215,13 +215,13 @@ rule verification_metrics_maps_baseline: ), truth=config["truth"]["root"], output: - OUT_ROOT / "data/baselines/{baseline_id}/metric_maps/{param}_{leadtime}.nc", + OUT_ROOT / "data/baselines/{baseline_id}/score_maps/{param}_{leadtime}.nc", params: baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: OUT_ROOT - / "logs/verification_metrics_maps_baseline/{baseline_id}-{param}-{leadtime}.log", + / "logs/verification_score_maps_baseline/{baseline_id}-{param}-{leadtime}.log", resources: cpus_per_task=24, mem_mb=50_000, diff --git a/workflow/scripts/plot_metric_maps.mo.py b/workflow/scripts/plot_score_maps.mo.py similarity index 100% rename from workflow/scripts/plot_metric_maps.mo.py rename to workflow/scripts/plot_score_maps.mo.py diff --git a/workflow/scripts/verification_metric_maps.py b/workflow/scripts/verification_score_maps.py similarity index 99% rename from workflow/scripts/verification_metric_maps.py rename to workflow/scripts/verification_score_maps.py index a5e0d9cd..a5675d29 100644 --- a/workflow/scripts/verification_metric_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -9,7 +9,7 @@ Usage ----- - uv run workflow/scripts/verification_metric_maps.py \\ + uv run workflow/scripts/verification_score_maps.py \\ output/data/runs/ \\ --truth /path/to/truth.zarr \\ --step 24 \\ @@ -557,7 +557,7 @@ def _strat_da(compute_fn) -> xr.DataArray: if args.output is None: source = args.run_root or args.baseline_root args.output = ( - source / f"verification_metric_maps_{args.param}_step{args.step:03d}h.nc" + source / f"verification_score_maps_{args.param}_step{args.step:03d}h.nc" ) main(args) diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index 2b57b96b..2e4fcb97 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -445,8 +445,46 @@ "title": "Locations", "type": "object" }, - "MetricMapsConfig": { - "description": "Parameters controlling which metric map plots are produced.", + "Profile": { + "description": "Workflow execution profile.", + "properties": { + "executor": { + "description": "Job executor, e.g. 'slurm'.", + "title": "Executor", + "type": "string" + }, + "global_resources": { + "$ref": "#/$defs/GlobalResources" + }, + "default_resources": { + "$ref": "#/$defs/DefaultResources" + }, + "jobs": { + "description": "Maximum number of parallel jobs.", + "minimum": 1, + "title": "Jobs", + "type": "integer" + }, + "batch_rules": { + "additionalProperties": { + "type": "integer" + }, + "description": "Define batches of the same rule that shall be executed within one job submission.", + "title": "Batch Rules", + "type": "object" + } + }, + "required": [ + "executor", + "global_resources", + "default_resources", + "jobs" + ], + "title": "Profile", + "type": "object" + }, + "ScoreMapsConfig": { + "description": "Parameters controlling which score map plots are produced.", "properties": { "params": { "default": [ @@ -548,45 +586,7 @@ "type": "array" } }, - "title": "MetricMapsConfig", - "type": "object" - }, - "Profile": { - "description": "Workflow execution profile.", - "properties": { - "executor": { - "description": "Job executor, e.g. 'slurm'.", - "title": "Executor", - "type": "string" - }, - "global_resources": { - "$ref": "#/$defs/GlobalResources" - }, - "default_resources": { - "$ref": "#/$defs/DefaultResources" - }, - "jobs": { - "description": "Maximum number of parallel jobs.", - "minimum": 1, - "title": "Jobs", - "type": "integer" - }, - "batch_rules": { - "additionalProperties": { - "type": "integer" - }, - "description": "Define batches of the same rule that shall be executed within one job submission.", - "title": "Batch Rules", - "type": "object" - } - }, - "required": [ - "executor", - "global_resources", - "default_resources", - "jobs" - ], - "title": "Profile", + "title": "ScoreMapsConfig", "type": "object" }, "Stratification": { @@ -731,9 +731,9 @@ "profile": { "$ref": "#/$defs/Profile" }, - "metric_maps": { - "$ref": "#/$defs/MetricMapsConfig", - "description": "Parameters for metric map plots (used with --maps flag)." + "score_maps": { + "$ref": "#/$defs/ScoreMapsConfig", + "description": "Parameters for score map plots (used with --maps flag)." } }, "required": [ From 77c9a64bb25053eff8e3e8b0fef955025f8947e1 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 4 Jun 2026 11:32:14 +0200 Subject: [PATCH 108/129] Add `leadtimes: "all"` shortcut to score_maps config Allows users to opt into computing scores for every leadtime in the union of all configured runs and baselines, without enumerating them. Default remains the explicit list, so a casual `--maps` invocation won't accidentally trigger heavy computation at high temporal resolution. - ScoreMapsConfig.leadtimes accepts `List[int] | Literal["all"]`. - New `resolve_leadtimes(spec)` helper in common.smk expands "all" to the union of `steps` from RUN_CONFIGS and BASELINE_CONFIGS. - Snakefile score_maps_all uses the resolver; no other rules change. - Example configs updated to show both forms (explicit list active, "all" alternative as a comment). --- config/forecasters-co1e.yaml | 5 +++-- config/forecasters-co2-disentangled.yaml | 5 +++-- config/forecasters-co2.yaml | 5 +++-- config/forecasters-ich1-oper-fixed.yaml | 5 +++-- config/forecasters-ich1-oper.yaml | 5 +++-- config/forecasters-ich1.yaml | 5 +++-- config/interpolators-co2.yaml | 5 +++-- config/interpolators-ich1.yaml | 5 +++-- src/evalml/config.py | 10 +++++++--- workflow/Snakefile | 4 ++-- workflow/rules/common.smk | 17 +++++++++++++++++ 11 files changed, 50 insertions(+), 21 deletions(-) diff --git a/config/forecasters-co1e.yaml b/config/forecasters-co1e.yaml index cf819cee..b0801581 100644 --- a/config/forecasters-co1e.yaml +++ b/config/forecasters-co1e.yaml @@ -83,8 +83,9 @@ score_maps: # - PS # - PMSL # - TOT_PREC - leadtimes: - - 24 + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" metrics: - BIAS # - RMSE diff --git a/config/forecasters-co2-disentangled.yaml b/config/forecasters-co2-disentangled.yaml index f5b27b2e..4f701f57 100644 --- a/config/forecasters-co2-disentangled.yaml +++ b/config/forecasters-co2-disentangled.yaml @@ -102,8 +102,9 @@ score_maps: # - PS # - PMSL # - TOT_PREC - leadtimes: - - 24 + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" metrics: - BIAS # - RMSE diff --git a/config/forecasters-co2.yaml b/config/forecasters-co2.yaml index 6dc0b6c7..87a28594 100644 --- a/config/forecasters-co2.yaml +++ b/config/forecasters-co2.yaml @@ -79,8 +79,9 @@ score_maps: # - PS # - PMSL # - TOT_PREC - leadtimes: - - 24 + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" metrics: - BIAS # - RMSE diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index c9c4b419..eb439b48 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -95,8 +95,9 @@ score_maps: # - PS # - PMSL # - TOT_PREC - leadtimes: - - 24 + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" metrics: - BIAS # - RMSE diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index a7a19710..626f56ad 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -90,8 +90,9 @@ score_maps: # - PS # - PMSL # - TOT_PREC - leadtimes: - - 24 + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" metrics: - BIAS # - RMSE diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index e27b5137..57f96404 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -102,8 +102,9 @@ score_maps: # - PS # - PMSL # - TOT_PREC - leadtimes: - - 24 + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" metrics: - BIAS # - RMSE diff --git a/config/interpolators-co2.yaml b/config/interpolators-co2.yaml index a718aea9..a25490c4 100644 --- a/config/interpolators-co2.yaml +++ b/config/interpolators-co2.yaml @@ -110,8 +110,9 @@ score_maps: # - PS # - PMSL # - TOT_PREC - leadtimes: - - 24 + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" metrics: - BIAS # - RMSE diff --git a/config/interpolators-ich1.yaml b/config/interpolators-ich1.yaml index 65c63b70..be522f53 100644 --- a/config/interpolators-ich1.yaml +++ b/config/interpolators-ich1.yaml @@ -99,8 +99,9 @@ score_maps: # - PS # - PMSL # - TOT_PREC - leadtimes: - - 24 + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" metrics: - BIAS # - RMSE diff --git a/src/evalml/config.py b/src/evalml/config.py index 82058452..bbaa513a 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, List, Any, ClassVar, FrozenSet +from typing import Dict, List, Any, ClassVar, FrozenSet, Literal from pydantic import BaseModel, Field, RootModel, field_validator @@ -218,9 +218,13 @@ class ScoreMapsConfig(BaseModel): "PS, PMSL, TOT_PREC (native), and SP_10M (derived wind speed from U_10M/V_10M)." ), ) - leadtimes: List[int] = Field( + leadtimes: List[int] | Literal["all"] = Field( default=list(range(6, 121, 6)), - description="List of lead times (hours) to plot.", + description=( + "List of lead times (hours) to plot, or the literal string 'all' " + "to expand to the union of step lists from all configured runs " + "and baselines." + ), ) metrics: List[str] = Field( default=["BIAS", "RMSE", "MAE"], diff --git a/workflow/Snakefile b/workflow/Snakefile index a542af4a..3152dc4a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -141,7 +141,7 @@ rule score_maps_all: expand( rules.plot_score_maps.output, run_id=collect_all_candidates(), - leadtime=config["score_maps"]["leadtimes"], + leadtime=resolve_leadtimes(config["score_maps"]["leadtimes"]), metric=config["score_maps"]["metrics"], param=config["score_maps"]["params"], region=config["score_maps"]["regions"], @@ -152,7 +152,7 @@ rule score_maps_all: expand( rules.plot_score_maps_baseline.output, baseline_id=list(BASELINE_CONFIGS), - leadtime=config["score_maps"]["leadtimes"], + leadtime=resolve_leadtimes(config["score_maps"]["leadtimes"]), metric=config["score_maps"]["metrics"], param=config["score_maps"]["params"], region=config["score_maps"]["regions"], diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index e858c80f..b44bde3d 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -299,3 +299,20 @@ RUN_CONFIGS = collect_all_runs() ENV_CONFIGS = collect_all_envs() BASELINE_CONFIGS = collect_all_baselines() EXPERIMENT_PARTICIPANTS = collect_experiment_participants() + + +def resolve_leadtimes(spec): + """Resolve a lead-time specification from config. + + Accepts: + - a list of ints — returned verbatim. + - the literal string "all" — expanded to the union of step lists + from all configured runs and baselines. + """ + if spec != "all": + return spec + all_steps = set() + for cfg in (*RUN_CONFIGS.values(), *BASELINE_CONFIGS.values()): + start, end, step = map(int, cfg["steps"].split("/")) + all_steps.update(range(start, end + 1, step)) + return sorted(all_steps) From eb0f5195670b8b4dfb3219ed84f047eaeba3f944 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 4 Jun 2026 11:58:57 +0200 Subject: [PATCH 109/129] Scope baseline score-maps paths by truth_label The same baseline (e.g. ICON-CH1-EPS) is now scored against different truths in different experiments (analyses, stations, low/high-res). Inserting the truth label into the path prevents output collisions without forcing recomputation for each experiment that shares the same (baseline, truth) combination. - New baseline output path: data/baselines/{baseline_id}/{truth_label}/score_maps/... - Mirror change in plot_score_maps_baseline's verif_file input. - Runs side unchanged: run_id is already hash-based per experiment. - Schema regenerated (was stale from the prior `leadtimes: "all"` commit). --- workflow/rules/plot.smk | 2 +- workflow/rules/verification.smk | 3 ++- workflow/tools/config.schema.json | 20 ++++++++++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 156c3a21..30c40aa0 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -181,7 +181,7 @@ use rule plot_score_maps as plot_score_maps_baseline with: input: script="workflow/scripts/plot_score_maps.mo.py", verif_file=OUT_ROOT - / "data/baselines/{baseline_id}/score_maps/{param}_{leadtime}.nc", + / f"data/baselines/{{baseline_id}}/{config['truth']['label']}/score_maps/{{param}}_{{leadtime}}.nc", output: OUT_ROOT / "results/{experiment}/score_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{init_hour}_{leadtime}.png", diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 80eca8b5..f541d0d1 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -215,7 +215,8 @@ rule verification_score_maps_baseline: ), truth=config["truth"]["root"], output: - OUT_ROOT / "data/baselines/{baseline_id}/score_maps/{param}_{leadtime}.nc", + OUT_ROOT + / f"data/baselines/{{baseline_id}}/{config['truth']['label']}/score_maps/{{param}}_{{leadtime}}.nc", params: baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index 2e4fcb97..9c508bcb 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -505,6 +505,18 @@ "type": "array" }, "leadtimes": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "const": "all", + "type": "string" + } + ], "default": [ 6, 12, @@ -527,12 +539,8 @@ 114, 120 ], - "description": "List of lead times (hours) to plot.", - "items": { - "type": "integer" - }, - "title": "Leadtimes", - "type": "array" + "description": "List of lead times (hours) to plot, or the literal string 'all' to expand to the union of step lists from all configured runs and baselines.", + "title": "Leadtimes" }, "metrics": { "default": [ From 7173cc22af8f31014072afeeaae07814d900d3a3 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 4 Jun 2026 12:09:36 +0200 Subject: [PATCH 110/129] Scope verification_score_maps_baseline log path by truth_label Consistency with the data path change in the previous commit. Two experiments verifying the same baseline against different truths would otherwise overwrite each other's logs. --- workflow/rules/verification.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index f541d0d1..8845db1f 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -222,7 +222,7 @@ rule verification_score_maps_baseline: reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: OUT_ROOT - / "logs/verification_score_maps_baseline/{baseline_id}-{param}-{leadtime}.log", + / f"logs/verification_score_maps_baseline/{{baseline_id}}-{config['truth']['label']}-{{param}}-{{leadtime}}.log", resources: cpus_per_task=24, mem_mb=50_000, From 7dfd8bea44fc71cac6d73e1db50954ab8e11d56a Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 4 Jun 2026 13:33:12 +0200 Subject: [PATCH 111/129] Add TODO note to consolidate truth-loading with data_input refactor Acknowledges duplication of ~80% of load_analysis_data_from_zarr in _open_zarr_component. Not consolidating here to avoid conflicts with the ongoing data_input refactor; the TODO points the right pointer at the eventual consolidation target. --- workflow/scripts/verification_score_maps.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index a5675d29..8071897c 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -84,6 +84,13 @@ def _compute_derived(ds: xr.Dataset, param: str) -> xr.DataArray: # --------------------------------------------------------------------------- # Truth loading # --------------------------------------------------------------------------- +# TODO: consolidate with src/data_input/__init__.py once the ongoing +# data-loading refactor lands. _open_zarr_component below duplicates +# ~80% of load_analysis_data_from_zarr but returns a lazy DataArray +# rather than a time-sliced Dataset, which is what our streaming +# aggregation needs. The right end-state is a shared lazy-open primitive +# in data_input that both consumers use; not introduced here because +# data_input is being reworked separately and we don't want to conflict. def _open_zarr_component(root: Path, param: str) -> xr.DataArray: From 3cda7782fb856da9ff831470af6533d4cebd6cc4 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 4 Jun 2026 13:39:16 +0200 Subject: [PATCH 112/129] Drop empty "GRIB step helpers" section header The helpers it used to label (TOT_PREC step disaggregation) moved into load_fct_data_from_grib when that logic was centralised. The header was left behind as dead decoration. --- workflow/scripts/verification_score_maps.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index 8071897c..aff57199 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -152,10 +152,6 @@ def open_truth_zarr(root: Path, param: str) -> xr.DataArray: return _open_zarr_component(root, param) -# --------------------------------------------------------------------------- -# GRIB step helpers -# --------------------------------------------------------------------------- - # --------------------------------------------------------------------------- # Init-time discovery # --------------------------------------------------------------------------- From 008c92ec0f216740120323b58fc38473802e5da5 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 4 Jun 2026 16:13:21 +0200 Subject: [PATCH 113/129] Derive baseline zarrs by glob instead of explicit CLI list verification_score_maps.py took both --baseline_root (to build per-reftime FCST.zarr paths) and --baseline_zarrs (an explicit list used only for init-time discovery). The list was redundant: the layout is fixed, so the script can glob FCST*.zarr under baseline_root itself, and discovered init times are filtered down to the configured reftimes anyway. - iter_baseline_init_times now globs baseline_root; drop --baseline_zarrs. - Keep baseline_zarrs as a Snakemake input: (dependency tracking re-runs the rule if the archive changes) but no longer pass it on the CLI. --- workflow/rules/verification.smk | 4 +++- workflow/scripts/verification_score_maps.py | 21 ++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 8845db1f..195ad2e5 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -208,6 +208,9 @@ rule verification_score_maps: rule verification_score_maps_baseline: input: script="workflow/scripts/verification_score_maps.py", + # Declared as inputs purely for dependency tracking (re-run if the baseline + # archive changes). The script discovers the zarrs itself by globbing + # `--baseline_root`, so this list is intentionally not passed on the CLI. baseline_zarrs=lambda wc: expand( "{root}/FCST{year}.zarr", root=BASELINE_CONFIGS[wc.baseline_id].get("root"), @@ -231,7 +234,6 @@ rule verification_score_maps_baseline: """ uv run {input.script} \ --baseline_root {params.baseline_root} \ - --baseline_zarrs {input.baseline_zarrs} \ --reftimes {params.reftimes} \ --truth {input.truth} \ --step {wildcards.leadtime} \ diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index aff57199..08259629 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -180,11 +180,17 @@ def iter_init_dirs(run_root: Path) -> list[tuple[datetime, Path]]: return result -def iter_baseline_init_times(zarr_paths: list[Path], step: int) -> list[datetime]: - """Return all init times from baseline zarr(s) that have the requested step available.""" +def iter_baseline_init_times(baseline_root: Path, step: int) -> list[datetime]: + """Return all init times from a baseline's zarr(s) that have the requested step available. + + The per-year zarrs are discovered by globbing ``FCST*.zarr`` under ``baseline_root`` + rather than taking an explicit list: the layout is fixed and the discovered init + times are filtered down to the configured reftimes by the caller anyway, so any + extra years picked up from the archive are harmless. + """ step_td = np.timedelta64(step, "h") reftimes = [] - for zarr_path in zarr_paths: + for zarr_path in sorted(baseline_root.glob("FCST*.zarr")): if not zarr_path.exists(): LOG.warning("Baseline zarr not found: %s", zarr_path) continue @@ -228,7 +234,7 @@ def main(args: Namespace) -> None: if args.baseline_root: init_items = [ (rt, None) - for rt in iter_baseline_init_times(args.baseline_zarrs, args.step) + for rt in iter_baseline_init_times(args.baseline_root, args.step) ] LOG.info("Found %d baseline init times", len(init_items)) else: @@ -512,13 +518,6 @@ def _strat_da(compute_fn) -> xr.DataArray: default=None, help="Root directory of a baseline (e.g. /path/to/ICON-CH1-EPS), containing FCST.zarr files.", ) - parser.add_argument( - "--baseline_zarrs", - type=Path, - nargs="+", - default=None, - help="Explicit list of baseline zarr paths (used by Snakemake for dependency tracking).", - ) parser.add_argument( "--truth", type=Path, From 082ff376fa27631811a0593022b7afb2f479dcc7 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 4 Jun 2026 16:48:26 +0200 Subject: [PATCH 114/129] Require --output for verification_score_maps, drop unsafe fallback The fallback derived the output path from --run_root/--baseline_root when --output was omitted. For baselines this would write into the read-only archive root (e.g. /store_new/mch/msopr/ml/ICON-CH1-EPS). Both workflow rules always pass --output explicitly, so the fallback was dead code; make --output required and remove it. --- workflow/scripts/verification_score_maps.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index 08259629..9fb3fa1c 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -548,7 +548,7 @@ def _strat_da(compute_fn) -> xr.DataArray: parser.add_argument( "--output", type=Path, - default=None, + required=True, help="Output NetCDF file.", ) args = parser.parse_args() @@ -556,10 +556,4 @@ def _strat_da(compute_fn) -> xr.DataArray: if bool(args.run_root) == bool(args.baseline_root): parser.error("Exactly one of --run_root or --baseline_root must be provided.") - if args.output is None: - source = args.run_root or args.baseline_root - args.output = ( - source / f"verification_score_maps_{args.param}_step{args.step:03d}h.nc" - ) - main(args) From 24ac031c78838d2396e547295be210670b2eb150 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 4 Jun 2026 17:06:58 +0200 Subject: [PATCH 115/129] Add STDE (standard deviation of error) score map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compute STDE = sqrt(E[e²] - E[e]²) per (season, lead) from the existing error accumulators, clamping the variance term at 0 to avoid spurious NaNs from float round-off. Update docstrings so the metric is discoverable; the plotter is generic and needs no change. --- workflow/scripts/plot_score_maps.mo.py | 2 +- workflow/scripts/verification_score_maps.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/workflow/scripts/plot_score_maps.mo.py b/workflow/scripts/plot_score_maps.mo.py index 2a41cd9e..26371865 100644 --- a/workflow/scripts/plot_score_maps.mo.py +++ b/workflow/scripts/plot_score_maps.mo.py @@ -56,7 +56,7 @@ def _(ArgumentParser, Path, np): parser.add_argument( "--metric", type=str, - help="Evaluation Metric. So far Bias, RMSE or MAE are implemented.", + help="Evaluation Metric. So far Bias, RMSE, MAE or STDE are implemented.", ) parser.add_argument("--season", type=str, default="all", help="season filter") parser.add_argument( diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index 9fb3fa1c..5f3afdaa 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -4,7 +4,7 @@ under a run directory, loads the corresponding GRIB forecast field and the matching truth slice from a reference zarr, maps the forecast onto the truth grid, and accumulates running error statistics without ever holding the full -time series in memory. The final BIAS / RMSE / MAE maps are written to a +time series in memory. The final BIAS / RMSE / MAE / STDE maps are written to a NetCDF file. Usage @@ -481,6 +481,19 @@ def _strat_da(compute_fn) -> xr.DataArray: f"{args.param}.MAE": _strat_da( lambda n, s, h: np.where(n > 0, accum_sum_ae[(s, h)] / n, np.nan) ), + f"{args.param}.STDE": _strat_da( + lambda n, s, h: np.where( + n > 0, + np.sqrt( + np.maximum( + accum_sum_se[(s, h)] / n + - (accum_sum_e[(s, h)] / n) ** 2, + 0.0, + ) + ), + np.nan, + ) + ), f"{args.param}.N": _strat_da(lambda n, s, h: np.where(n > 0, n, np.nan)), }, attrs={ From 91983fe5fe3bd46f6f8b9bd55cfe12492bbfaa8e Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 4 Jun 2026 17:50:52 +0200 Subject: [PATCH 116/129] =?UTF-8?q?Rename=20score-maps=20"metric"=20select?= =?UTF-8?q?or=20=E2=86=92=20"score"=20for=20naming=20consistency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes commit e9fbef9, which renamed the "...maps" containers but left the per-map score selector still called "metric". By evalml's convention BIAS/RMSE/MAE/STDE compare forecast to truth, so they are scores. Renames within the score_maps feature only: - ScoreMapsConfig.metrics -> scores (+ regenerated config.schema.json) - score_maps config key metrics: -> scores: in all config/*.yaml - Snakefile expand kwarg + plot.smk {metric} wildcard / --metric flag - plot_score_maps.mo.py: --score flag, variables, helper, title The main-side verification_metrics* pipeline is left as-is, consistent with e9fbef9's scope note. Computation and plotting of metric maps like min, max, mean is deferred to a follow-up PR. --- config/forecasters-co1e.yaml | 2 +- config/forecasters-co2-disentangled.yaml | 2 +- config/forecasters-co2.yaml | 2 +- config/forecasters-ich1-oper-fixed.yaml | 2 +- config/forecasters-ich1-oper.yaml | 2 +- config/forecasters-ich1.yaml | 2 +- config/interpolators-co2.yaml | 2 +- config/interpolators-ich1.yaml | 2 +- src/evalml/config.py | 4 ++-- workflow/Snakefile | 4 ++-- workflow/rules/plot.smk | 12 +++++----- workflow/scripts/plot_score_maps.mo.py | 28 ++++++++++++------------ workflow/tools/config.schema.json | 6 ++--- 13 files changed, 35 insertions(+), 35 deletions(-) diff --git a/config/forecasters-co1e.yaml b/config/forecasters-co1e.yaml index b0801581..03fcfcfe 100644 --- a/config/forecasters-co1e.yaml +++ b/config/forecasters-co1e.yaml @@ -86,7 +86,7 @@ score_maps: leadtimes: [6, 24] # Or, to compute every available leadtime from runs+baselines: # leadtimes: "all" - metrics: + scores: - BIAS # - RMSE # - MAE diff --git a/config/forecasters-co2-disentangled.yaml b/config/forecasters-co2-disentangled.yaml index 4f701f57..91489c00 100644 --- a/config/forecasters-co2-disentangled.yaml +++ b/config/forecasters-co2-disentangled.yaml @@ -105,7 +105,7 @@ score_maps: leadtimes: [6, 24] # Or, to compute every available leadtime from runs+baselines: # leadtimes: "all" - metrics: + scores: - BIAS # - RMSE # - MAE diff --git a/config/forecasters-co2.yaml b/config/forecasters-co2.yaml index 87a28594..ccf67396 100644 --- a/config/forecasters-co2.yaml +++ b/config/forecasters-co2.yaml @@ -82,7 +82,7 @@ score_maps: leadtimes: [6, 24] # Or, to compute every available leadtime from runs+baselines: # leadtimes: "all" - metrics: + scores: - BIAS # - RMSE # - MAE diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index eb439b48..09261399 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -98,7 +98,7 @@ score_maps: leadtimes: [6, 24] # Or, to compute every available leadtime from runs+baselines: # leadtimes: "all" - metrics: + scores: - BIAS # - RMSE # - MAE diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 626f56ad..1f53a534 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -93,7 +93,7 @@ score_maps: leadtimes: [6, 24] # Or, to compute every available leadtime from runs+baselines: # leadtimes: "all" - metrics: + scores: - BIAS # - RMSE # - MAE diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index 57f96404..09fbe0af 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -105,7 +105,7 @@ score_maps: leadtimes: [6, 24] # Or, to compute every available leadtime from runs+baselines: # leadtimes: "all" - metrics: + scores: - BIAS # - RMSE # - MAE diff --git a/config/interpolators-co2.yaml b/config/interpolators-co2.yaml index a25490c4..64c3f30f 100644 --- a/config/interpolators-co2.yaml +++ b/config/interpolators-co2.yaml @@ -113,7 +113,7 @@ score_maps: leadtimes: [6, 24] # Or, to compute every available leadtime from runs+baselines: # leadtimes: "all" - metrics: + scores: - BIAS # - RMSE # - MAE diff --git a/config/interpolators-ich1.yaml b/config/interpolators-ich1.yaml index be522f53..833ed22d 100644 --- a/config/interpolators-ich1.yaml +++ b/config/interpolators-ich1.yaml @@ -102,7 +102,7 @@ score_maps: leadtimes: [6, 24] # Or, to compute every available leadtime from runs+baselines: # leadtimes: "all" - metrics: + scores: - BIAS # - RMSE # - MAE diff --git a/src/evalml/config.py b/src/evalml/config.py index bbaa513a..73925af8 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -226,9 +226,9 @@ class ScoreMapsConfig(BaseModel): "and baselines." ), ) - metrics: List[str] = Field( + scores: List[str] = Field( default=["BIAS", "RMSE", "MAE"], - description="List of verification metrics to plot.", + description="List of verification scores to plot.", ) regions: List[str] = Field( default=["switzerland", "centraleurope"], diff --git a/workflow/Snakefile b/workflow/Snakefile index 3152dc4a..d8001d2a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -142,7 +142,7 @@ rule score_maps_all: rules.plot_score_maps.output, run_id=collect_all_candidates(), leadtime=resolve_leadtimes(config["score_maps"]["leadtimes"]), - metric=config["score_maps"]["metrics"], + score=config["score_maps"]["scores"], param=config["score_maps"]["params"], region=config["score_maps"]["regions"], season=config["score_maps"]["seasons"], @@ -153,7 +153,7 @@ rule score_maps_all: rules.plot_score_maps_baseline.output, baseline_id=list(BASELINE_CONFIGS), leadtime=resolve_leadtimes(config["score_maps"]["leadtimes"]), - metric=config["score_maps"]["metrics"], + score=config["score_maps"]["scores"], param=config["score_maps"]["params"], region=config["score_maps"]["regions"], season=config["score_maps"]["seasons"], diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 30c40aa0..8f48eed2 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -151,13 +151,13 @@ rule plot_score_maps: verif_file=OUT_ROOT / "data/runs/{run_id}/score_maps/{param}_{leadtime}.nc", output: OUT_ROOT - / "results/{experiment}/score_maps/runs/{run_id}/{param}_{metric}_{region}_{season}_{init_hour}_{leadtime}.png", + / "results/{experiment}/score_maps/runs/{run_id}/{param}_{score}_{region}_{season}_{init_hour}_{leadtime}.png", wildcard_constraints: leadtime=r"\d+", # only digits init_hour=r"all|\d{1,2}", log: OUT_ROOT - / "logs/plot_score_maps/{experiment}/{run_id}-{param}-{metric}-{region}-{season}-{init_hour}-{leadtime}.log", + / "logs/plot_score_maps/{experiment}/{run_id}-{param}-{score}-{region}-{season}-{init_hour}-{leadtime}.log", resources: slurm_partition="postproc", cpus_per_task=1, @@ -167,12 +167,12 @@ rule plot_score_maps: export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) uv run python {input.script} \ --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ - --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ + --param {wildcards.param} --leadtime {wildcards.leadtime} --score {wildcards.score} \ --season {wildcards.season} --init_hour {wildcards.init_hour} > {log} 2>&1 # interactive editing (needs to set localrule: True and use only one core) # marimo edit {input.script} -- \ # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ - # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ + # --param {wildcards.param} --leadtime {wildcards.leadtime} --score {wildcards.score} \ # --season {wildcards.season} --init_hour {wildcards.init_hour} """ @@ -184,7 +184,7 @@ use rule plot_score_maps as plot_score_maps_baseline with: / f"data/baselines/{{baseline_id}}/{config['truth']['label']}/score_maps/{{param}}_{{leadtime}}.nc", output: OUT_ROOT - / "results/{experiment}/score_maps/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{init_hour}_{leadtime}.png", + / "results/{experiment}/score_maps/baselines/{baseline_id}/{param}_{score}_{region}_{season}_{init_hour}_{leadtime}.png", log: OUT_ROOT - / "logs/plot_score_maps/{experiment}/{baseline_id}-{param}-{metric}-{region}-{season}-{init_hour}-{leadtime}.log", + / "logs/plot_score_maps/{experiment}/{baseline_id}-{param}-{score}-{region}-{season}-{init_hour}-{leadtime}.log", diff --git a/workflow/scripts/plot_score_maps.mo.py b/workflow/scripts/plot_score_maps.mo.py index 26371865..c484ad3e 100644 --- a/workflow/scripts/plot_score_maps.mo.py +++ b/workflow/scripts/plot_score_maps.mo.py @@ -54,9 +54,9 @@ def _(ArgumentParser, Path, np): parser.add_argument("--param", type=str, help="parameter") parser.add_argument("--region", type=str, help="name of region") parser.add_argument( - "--metric", + "--score", type=str, - help="Evaluation Metric. So far Bias, RMSE, MAE or STDE are implemented.", + help="Evaluation Score. So far Bias, RMSE, MAE or STDE are implemented.", ) parser.add_argument("--season", type=str, default="all", help="season filter") parser.add_argument( @@ -71,7 +71,7 @@ def _(ArgumentParser, Path, np): region = args.region season = args.season init_hour = args.init_hour - metric = args.metric + score = args.score if isinstance(init_hour, str): if init_hour == "all": @@ -86,20 +86,20 @@ def _(ArgumentParser, Path, np): return ( init_hour, lead_time, - metric, outfn, param, region, + score, season, verif_file, ) @app.cell -def _(LOG, init_hour, metric, param, season, verif_file, xr): +def _(LOG, init_hour, param, score, season, verif_file, xr): ds = xr.open_dataset(verif_file) LOG.info("Opened dataset: %s", ds) - var = f"{param}.{metric}" + var = f"{param}.{score}" LOG.info( "Selecting variable '%s' for season '%s', init_hour=%s", var, season, init_hour ) @@ -118,15 +118,15 @@ def _(LOG, init_hour, metric, param, season, verif_file, xr): @app.cell def _(CMAP_DEFAULTS, ekp): - def get_style(param, metric, units_override=None): + def get_style(param, score, units_override=None): """Get style and colormap settings for the plot. Needed because cmap/norm does not work in Style(colors=cmap), still needs to be passed as arguments to tripcolor()/tricontourf(). """ - metric_key = f"{param}.{metric}.map" + score_key = f"{param}.{score}.map" cfg = ( - CMAP_DEFAULTS[metric_key] - if metric_key in CMAP_DEFAULTS + CMAP_DEFAULTS[score_key] + if score_key in CMAP_DEFAULTS else CMAP_DEFAULTS.get(param, {}) ) units = units_override if units_override is not None else cfg.get("units", "") @@ -157,11 +157,11 @@ def _( get_style, init_hour, lead_time, - metric, np, outfn, param, region, + score, season, ): # plot individual fields @@ -183,14 +183,14 @@ def _( plot_vals = ds.values.ravel() - style_kwargs = get_style(param, metric) + style_kwargs = get_style(param, score) LOG.info("style_kwargs: %s", style_kwargs) if np.all(np.isnan(plot_vals)): LOG.warning( "All values are NaN for %s %s season=%s — plotting empty map.", param, - metric, + score, season, ) import matplotlib.patches as mpatches @@ -209,7 +209,7 @@ def _( init_hour_lbl = "all" if init_hour == -999 else f"{init_hour:02d}" fig.title( - f"{metric} of {param}, Season: {season}, " + f"{score} of {param}, Season: {season}, " f"Init hour: {init_hour_lbl}, Lead Time: {lead_time}" ) diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index 9c508bcb..40e2936f 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -542,17 +542,17 @@ "description": "List of lead times (hours) to plot, or the literal string 'all' to expand to the union of step lists from all configured runs and baselines.", "title": "Leadtimes" }, - "metrics": { + "scores": { "default": [ "BIAS", "RMSE", "MAE" ], - "description": "List of verification metrics to plot.", + "description": "List of verification scores to plot.", "items": { "type": "string" }, - "title": "Metrics", + "title": "Scores", "type": "array" }, "regions": { From 1b92a4ea901459d77cb835d2295f9c58ddf93df6 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 9 Jun 2026 16:33:00 +0200 Subject: [PATCH 117/129] Fix TOT_PREC score-map accumulation window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Score maps verified a 0→step forecast sum against the TOT_PREC_H period-accumulation truth. De-accumulate over the requested steps instead and have the caller request the [step − period, step] window (period derived from the configured step spacing, not hardcoded 6h), selecting the matching TOT_PREC_H truth. Keep IC (step 0 = 0) synthesis when step 0 is in the window but omitted by anemoi-inference. --- src/data_input/__init__.py | 129 ++++++++++---------- workflow/rules/verification.smk | 3 + workflow/scripts/verification_score_maps.py | 95 +++++++++++--- 3 files changed, 144 insertions(+), 83 deletions(-) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index 71e49bf6..7da4954d 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -118,21 +118,7 @@ def load_fct_data_from_grib( # adds noticeable overhead to verif_metrics and the plot rules. files = sorted(root.glob(f"{reftime:%Y%m%d%H%M}*.grib")) fds = data_source.FileDataSource(datafiles=files) - # For TOT_PREC (cumulative-from-start) we need step 0 to disaggregate to a - # 0->step period accumulation even when the caller asks for a single step. - # anemoi-inference may omit step 0 from the GRIB; tolerate that and - # synthesize lead_time=0, TOT_PREC=0 below (cumulative-from-start has - # nothing accumulated at the IC by definition). - needs_step_zero = "TOT_PREC" in params and 0 not in steps - fetch_steps = [0, *steps] if needs_step_zero else list(steps) - ds = grib_decoder.load(fds, {"param": params, "step": fetch_steps}) - # grib_decoder.load may silently drop steps that aren't on disk - # (anemoi-inference often omits step 0 even with cumulative-from-start - # accumulation). Detect that here so the TOT_PREC block can synthesize - # lead_time=0, TOT_PREC=0 below. - zero_lt = np.timedelta64(0, "h") - loaded_lead_times = next(iter(ds.values())).lead_time.values - step_zero_synthetic = needs_step_zero and zero_lt not in loaded_lead_times + ds = grib_decoder.load(fds, {"param": params, "step": list(steps)}) for var, da in ds.items(): if "z" in da.dims and da.sizes["z"] == 1: ds[var] = da.squeeze("z", drop=True) @@ -140,37 +126,35 @@ def load_fct_data_from_grib( ds[var] = da.rename({"z": da.attrs["vcoord_type"]}) ds = xr.merge([ds[p].rename(p) for p in ds], compat="no_conflicts") lead_times = np.array(steps, dtype="timedelta64[h]") - fetch_lead_times = np.array(fetch_steps, dtype="timedelta64[h]") - # Restrict to the lead times we'll work with (fetch_lead_times = requested - # steps + step 0 if needed). This drops any extra (e.g. hourly) steps the - # GRIB may contain beyond what we asked for — e.g. when consuming output - # from an interpolator emulator or a baseline with sub-step output. - if step_zero_synthetic: - # Step 0 is missing from the GRIB; reindex inserts NaN at lead 0, - # which the xr.where below replaces with 0. - ds = ds.sel(lead_time=lead_times).reindex(lead_time=fetch_lead_times) - else: - ds = ds.sel(lead_time=fetch_lead_times) + zero_lt = np.timedelta64(0, "h") if "TOT_PREC" in ds.data_vars: ## Disaggregate TOT_PREC from cumulative-from-start (expected when the ## accumulate_from_start_of_forecast post-processor is enabled in - ## anemoi-inference) to per-step accumulations. + ## anemoi-inference) to per-step accumulations. The accumulation period + ## is set by the requested steps: to obtain the period accumulation + ## ending at `step`, the caller must request [step - period, step]. The + ## first requested step has no predecessor and becomes NaN. ## - ## anemoi-inference sometimes omits step 0 from the GRIB even with - ## accumulate_from_start_of_forecast enabled. After the outer-join - ## merge above, lead_time=0 of TOT_PREC is then NaN, which would - ## propagate through .diff() and wipe out the first period - ## accumulation. Set it explicitly to 0 (cumulative-from-start has - ## nothing accumulated at the forecast initial time by definition). - ## Restricting to lead_time=0 leaves any other NaNs (e.g. from - ## boundary-trim masks) untouched. - ds = ds.assign( - TOT_PREC=xr.where( - ds.lead_time == np.timedelta64(0, "h"), - 0.0, - ds.TOT_PREC, + ## Align to the requested steps, dropping any extra (e.g. hourly) steps + ## the GRIB may contain — e.g. output from an interpolator emulator or a + ## baseline with sub-step output. anemoi-inference sometimes omits step + ## 0 (the initial condition) from the GRIB even with + ## accumulate_from_start_of_forecast enabled; diffing against a + ## missing/NaN step 0 would wipe out the first period accumulation, so + ## when step 0 is requested but absent we reindex it in (it is set to 0 + ## just below). + if zero_lt in lead_times and zero_lt not in ds.lead_time.values: + ds = ds.reindex(lead_time=lead_times) + else: + ds = ds.sel(lead_time=lead_times) + if zero_lt in lead_times: + ## Step 0 is the forecast initial time: cumulative-from-start has + ## nothing accumulated there by definition. Restricting to + ## lead_time=0 leaves any other NaNs (e.g. boundary-trim masks) + ## untouched. + ds = ds.assign( + TOT_PREC=xr.where(ds.lead_time == zero_lt, 0.0, ds.TOT_PREC) ) - ) ## Sanity-check that the incoming data is actually cumulative: if ## .diff() produces significantly negative values, TOT_PREC is already ## period-accumulated and a second disaggregation would produce @@ -186,13 +170,13 @@ def load_fct_data_from_grib( f"post-processor is enabled in the anemoi-inference config " f"for this source." ) - ## .diff() drops lead_time=0; .reindex() restores it as NaN (no - ## accumulation period exists at the forecast initial time). Clip - ## small float-noise negatives to zero (anything below -0.1 mm has - ## already been caught by the check above). + ## .diff() drops the first requested step; .reindex() restores it as + ## NaN (no accumulation period exists before it). Clip small + ## float-noise negatives to zero (anything below -0.1 mm has already + ## been caught by the check above). ds = ds.assign(TOT_PREC=diff.clip(min=0.0).reindex(lead_time=lead_times)) - # Drop the auxiliary step 0 from any non-TOT_PREC variables. - ds = ds.sel(lead_time=lead_times) + else: + ds = ds.sel(lead_time=lead_times) # make sure time coordinate is available, and valid_time is not if "valid_time" in ds.coords: ds = ds.rename({"valid_time": "time"}) @@ -219,25 +203,36 @@ def load_baseline_from_zarr( {"forecast_reference_time": "ref_time", "step": "lead_time"} ).sortby("lead_time") lead_times = np.array(steps, dtype="timedelta64[h]") - # For TOT_PREC (cumulative-from-start) we need step 0 in the slice so that - # .diff() yields a 0->step period accumulation even when the caller - # requested a single step. The extra step is dropped at the final reindex. zero_lt = np.timedelta64(0, "h") - if "TOT_PREC" in params and zero_lt not in lead_times: - fetch_lead_times = np.concatenate([[zero_lt], lead_times]) - else: - fetch_lead_times = lead_times - # Restrict to the requested lead times up-front so that the TOT_PREC - # disaggregation below operates on the correct step interval, and so that - # all other variables avoid loading unused hourly steps from the zarr. - baseline = baseline[params].sel(ref_time=reftime, lead_time=fetch_lead_times) + baseline = baseline[params].sel(ref_time=reftime) if "TOT_PREC" in baseline.data_vars: + ## Disaggregate TOT_PREC from cumulative-from-start (the expected zarr + ## convention for processed NWP output) to per-step accumulations. The + ## accumulation period is set by the requested steps: to obtain the + ## period accumulation ending at `step`, the caller must request + ## [step - period, step]. The first requested step has no predecessor + ## and becomes NaN. + ## + ## Align to the requested steps up-front so the disaggregation operates + ## on the correct interval and other variables avoid loading unused + ## hourly steps. If step 0 (the initial time) is requested but absent, + ## reindex it in so it can be set to 0 below — diffing against a + ## missing/NaN step 0 would otherwise wipe out the first accumulation. + if zero_lt in lead_times and zero_lt not in baseline.lead_time.values: + baseline = baseline.reindex(lead_time=lead_times) + else: + baseline = baseline.sel(lead_time=lead_times) if baseline.TOT_PREC.units == "m": baseline = baseline.assign(TOT_PREC=lambda x: x.TOT_PREC * 1000) baseline.TOT_PREC.attrs["units"] = "kg m-2" - ## Disaggregate TOT_PREC from cumulative-from-start (the expected zarr - ## convention for processed NWP output) to per-step accumulations. - ## + if zero_lt in lead_times: + ## Step 0 is the forecast initial time: cumulative-from-start has + ## nothing accumulated there by definition. + baseline = baseline.assign( + TOT_PREC=xr.where( + baseline.lead_time == zero_lt, 0.0, baseline.TOT_PREC + ) + ) ## Sanity-check that the incoming data is actually cumulative: if ## .diff() produces significantly negative values, TOT_PREC is already ## period-accumulated and a second disaggregation would produce @@ -250,15 +245,15 @@ def load_baseline_from_zarr( f"TOT_PREC in the baseline zarr appears to already be " f"period-accumulated (min(.diff()) = {min_diff:.3e} m)." ) - ## .diff() drops lead_time=0; .reindex() restores it as NaN (no - ## accumulation period exists at the forecast initial time). Clip - ## small float-noise negatives to zero (anything below -0.1 mm has - ## already been caught by the check above). + ## .diff() drops the first requested step; .reindex() restores it as + ## NaN (no accumulation period exists before it). Clip small + ## float-noise negatives to zero (anything below -0.1 mm has already + ## been caught by the check above). baseline = baseline.assign( TOT_PREC=diff.clip(min=0.0).reindex(lead_time=lead_times) ) - # Drop the auxiliary step 0 from any non-TOT_PREC variables. - baseline = baseline.sel(lead_time=lead_times) + else: + baseline = baseline.sel(lead_time=lead_times) baseline = baseline.assign_coords(time=baseline.ref_time + baseline.lead_time) if "latitude" in baseline.coords and "longitude" in baseline: baseline = baseline.rename({"latitude": "lat", "longitude": "lon"}) diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 195ad2e5..07fe3289 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -200,6 +200,7 @@ rule verification_score_maps: --reftimes {params.reftimes} \ --truth {input.truth} \ --step {wildcards.leadtime} \ + --steps "{params.fcst_steps}" \ --param {wildcards.param} \ --output {output} > {log} 2>&1 """ @@ -222,6 +223,7 @@ rule verification_score_maps_baseline: / f"data/baselines/{{baseline_id}}/{config['truth']['label']}/score_maps/{{param}}_{{leadtime}}.nc", params: baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), + baseline_steps=lambda wc: BASELINE_CONFIGS[wc.baseline_id]["steps"], reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: OUT_ROOT @@ -237,6 +239,7 @@ rule verification_score_maps_baseline: --reftimes {params.reftimes} \ --truth {input.truth} \ --step {wildcards.leadtime} \ + --steps "{params.baseline_steps}" \ --param {wildcards.param} \ --output {output} > {log} 2>&1 """ diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index 5f3afdaa..708d2008 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -24,7 +24,7 @@ import numpy as np import xarray as xr -from data_input import load_fct_data_from_grib, load_baseline_from_zarr +from data_input import load_fct_data_from_grib, load_baseline_from_zarr, parse_steps from verification.spatial import map_forecast_to_truth LOG = logging.getLogger(__name__) @@ -62,16 +62,28 @@ def _season_of(dt: datetime) -> str: "PMSL": "msl", "TOT_PREC": "tp", } -_PARAMS_MAP_CO1 = {k: k.replace("TOT_PREC", "TOT_PREC_6H") for k in _PARAMS_MAP_CO2} - # Derived variables and the components they require. _DERIVED = { "SP_10M": ("U_10M", "V_10M"), } +# Params whose GRIB/zarr values are cumulative-from-start accumulations and must +# be de-accumulated over a [step - period, step] window before verification. +_ACCUMULATED_PARAMS = {"TOT_PREC"} + -def _params_map(truth_root: Path) -> dict[str, str]: - return _PARAMS_MAP_CO2 if "co2" in truth_root.name else _PARAMS_MAP_CO1 +def _params_map(truth_root: Path, accum_h: int | None = None) -> dict[str, str]: + """Map canonical parameter names to truth-zarr variable names. + + COSMO-2e zarrs use short CF names. COSMO-1e / ICON zarrs store precip as + period accumulations named ``TOT_PREC_H``, where N is the accumulation + length in hours (matching the verification step spacing); pass it via + ``accum_h``. + """ + if "co2" in truth_root.name: + return _PARAMS_MAP_CO2 + suffix = f"TOT_PREC_{accum_h}H" if accum_h else "TOT_PREC_6H" + return {k: k.replace("TOT_PREC", suffix) for k in _PARAMS_MAP_CO2} def _compute_derived(ds: xr.Dataset, param: str) -> xr.DataArray: @@ -93,9 +105,11 @@ def _compute_derived(ds: xr.Dataset, param: str) -> xr.DataArray: # data_input is being reworked separately and we don't want to conflict. -def _open_zarr_component(root: Path, param: str) -> xr.DataArray: +def _open_zarr_component( + root: Path, param: str, accum_h: int | None = None +) -> xr.DataArray: """Open a single native zarr variable lazily as a DataArray.""" - zarr_param = _params_map(root)[param] + zarr_param = _params_map(root, accum_h)[param] ds = xr.open_zarr(root, consolidated=False) ds = ds.set_index(time="dates") @@ -135,21 +149,26 @@ def _open_zarr_component(root: Path, param: str) -> xr.DataArray: return da -def open_truth_zarr(root: Path, param: str) -> xr.DataArray: +def open_truth_zarr( + root: Path, param: str, accum_h: int | None = None +) -> xr.DataArray: """Open the truth zarr lazily and return a DataArray for *param*. For derived variables (e.g. SP_10M) the required components are loaded and the derivation is applied on the fly. The returned DataArray has dimensions ``(time, y, x)`` or ``(time, values)`` and always exposes ``lat``/``lon``. + ``accum_h`` selects the precip accumulation length (TOT_PREC_H). """ if param in _DERIVED: components = { - c: _open_zarr_component(root, c).drop_vars("variable", errors="ignore") + c: _open_zarr_component(root, c, accum_h).drop_vars( + "variable", errors="ignore" + ) for c in _DERIVED[param] } ds = xr.Dataset(components) return _compute_derived(ds, param) - return _open_zarr_component(root, param) + return _open_zarr_component(root, param, accum_h) # --------------------------------------------------------------------------- @@ -217,8 +236,38 @@ def main(args: Namespace) -> None: LOG.info("Output : %s", args.output) LOG.info("=" * 60) + # Accumulated params (TOT_PREC) are stored cumulative-from-start, while the + # truth is a period accumulation whose length equals the verification step + # spacing (e.g. 6h for steps "0/120/6"). Derive that period so we can (a) + # request the matching [step - period, step] window from the forecast loader + # and (b) read the matching TOT_PREC_H truth variable. We do not + # assume a fixed period; it follows the configured --steps. + accum_h: int | None = None + if args.param in _ACCUMULATED_PARAMS: + if not args.steps: + raise ValueError( + f"--steps is required for accumulated param '{args.param}' " + "(used to derive the accumulation period)." + ) + spacing = np.diff(parse_steps(args.steps)) + if spacing.size == 0: + raise ValueError( + f"Cannot derive an accumulation period from --steps '{args.steps}'." + ) + accum_h = int(spacing.min()) + if args.step < accum_h: + raise ValueError( + f"Lead time {args.step}h is smaller than the {accum_h}h " + f"accumulation period; cannot form a [step - period, step] " + f"window for '{args.param}'." + ) + req_steps = [args.step - accum_h, args.step] + LOG.info("Accumulation period: %dh (forecast window %s)", accum_h, req_steps) + else: + req_steps = [args.step] + # Open the truth zarr once; individual time slices are loaded on demand. - truth_da = open_truth_zarr(args.truth, args.param) + truth_da = open_truth_zarr(args.truth, args.param, accum_h) # Normalise to datetime64[ns] so membership checks work regardless of zarr precision. truth_da = truth_da.assign_coords( time=truth_da.time.values.astype("datetime64[ns]") @@ -299,17 +348,19 @@ def main(args: Namespace) -> None: fcst = load_baseline_from_zarr( root=zarr_path, reftime=reftime, - steps=[args.step], + steps=req_steps, params=fct_params, ) else: - # The loaders handle cumulative-from-start disaggregation - # internally (including fetching step 0 when needed for - # TOT_PREC), so a single-step request is sufficient here. + # For accumulated params (TOT_PREC) req_steps is the + # [step - period, step] window; the loader de-accumulates the + # cumulative-from-start field over those steps, yielding the + # period accumulation at `step`. Instantaneous params request a + # single step. The target step is selected just below. fcst = load_fct_data_from_grib( root=grib_dir, reftime=reftime, - steps=[args.step], + steps=req_steps, params=fct_params, ) except Exception as exc: @@ -549,6 +600,18 @@ def _strat_da(compute_fn) -> xr.DataArray: required=True, help="Variable to verify (e.g. T_2M, TD_2M, U_10M).", ) + parser.add_argument( + "--steps", + type=str, + default=None, + help=( + "Forecast step spec 'start/stop/step' (e.g. '0/120/6'). Required for " + "accumulated params (TOT_PREC): the accumulation period is the step " + "spacing, the forecast is accumulated over [step - period, step], and " + "the matching TOT_PREC_H truth variable is read. Ignored for " + "instantaneous params." + ), + ) parser.add_argument( "--reftimes", nargs="+", From 78cf10538accb63096787d73d71d557682c06246 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Tue, 9 Jun 2026 18:51:14 +0200 Subject: [PATCH 118/129] Apply ruff-format and snakefmt formatting after merge --- workflow/rules/plot.smk | 8 +++---- workflow/rules/verification.smk | 24 ++++++++++----------- workflow/scripts/verification_score_maps.py | 10 +++------ 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 556474ba..1392a53d 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -174,12 +174,12 @@ rule plot_score_maps: output: OUT_ROOT / "results/{experiment}/score_maps/runs/{run_id}/{param}_{score}_{region}_{season}_{init_hour}_{leadtime}.png", - wildcard_constraints: - leadtime=r"\d+", # only digits - init_hour=r"all|\d{1,2}", log: OUT_ROOT / "logs/plot_score_maps/{experiment}/{run_id}-{param}-{score}-{region}-{season}-{init_hour}-{leadtime}.log", + wildcard_constraints: + leadtime=r"\d+", # only digits + init_hour=r"all|\d{1,2}", resources: slurm_partition="postproc", cpus_per_task=1, @@ -190,7 +190,7 @@ rule plot_score_maps: uv run python {input.script} \ --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ --param {wildcards.param} --leadtime {wildcards.leadtime} --score {wildcards.score} \ - --season {wildcards.season} --init_hour {wildcards.init_hour} > {log} 2>&1 + --season {wildcards.season} --init_hour {wildcards.init_hour} >{log} 2>&1 # interactive editing (needs to set localrule: True and use only one core) # marimo edit {input.script} -- \ # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index abeb8492..6320d296 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -184,6 +184,12 @@ rule verification_score_maps: truth=config["truth"]["root"], output: OUT_ROOT / "data/runs/{run_id}/score_maps/{param}_{leadtime}.nc", + log: + OUT_ROOT / "logs/verification_score_maps/{run_id}-{param}-{leadtime}.log", + resources: + cpus_per_task=24, + mem_mb=50_000, + runtime="60m", # wildcard_constraints: # run_id="^" # to avoid ambiguitiy with run_baseline_verif # TODO: implement logic to use experiment name instead of run_id as wildcard @@ -192,12 +198,6 @@ rule verification_score_maps: fcst_steps=lambda wc: RUN_CONFIGS[wc.run_id]["steps"], truth_label=config["truth"]["label"], reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), - log: - OUT_ROOT / "logs/verification_score_maps/{run_id}-{param}-{leadtime}.log", - resources: - cpus_per_task=24, - mem_mb=50_000, - runtime="60m", shell: """ uv run {input.script} \ @@ -207,7 +207,7 @@ rule verification_score_maps: --step {wildcards.leadtime} \ --steps "{params.fcst_steps}" \ --param {wildcards.param} \ - --output {output} > {log} 2>&1 + --output {output} >{log} 2>&1 """ @@ -226,10 +226,6 @@ rule verification_score_maps_baseline: output: OUT_ROOT / f"data/baselines/{{baseline_id}}/{config['truth']['label']}/score_maps/{{param}}_{{leadtime}}.nc", - params: - baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), - baseline_steps=lambda wc: BASELINE_CONFIGS[wc.baseline_id]["steps"], - reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), log: OUT_ROOT / f"logs/verification_score_maps_baseline/{{baseline_id}}-{config['truth']['label']}-{{param}}-{{leadtime}}.log", @@ -237,6 +233,10 @@ rule verification_score_maps_baseline: cpus_per_task=24, mem_mb=50_000, runtime="60m", + params: + baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), + baseline_steps=lambda wc: BASELINE_CONFIGS[wc.baseline_id]["steps"], + reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), shell: """ uv run {input.script} \ @@ -246,5 +246,5 @@ rule verification_score_maps_baseline: --step {wildcards.leadtime} \ --steps "{params.baseline_steps}" \ --param {wildcards.param} \ - --output {output} > {log} 2>&1 + --output {output} >{log} 2>&1 """ diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index 708d2008..b5b35fd9 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -149,9 +149,7 @@ def _open_zarr_component( return da -def open_truth_zarr( - root: Path, param: str, accum_h: int | None = None -) -> xr.DataArray: +def open_truth_zarr(root: Path, param: str, accum_h: int | None = None) -> xr.DataArray: """Open the truth zarr lazily and return a DataArray for *param*. For derived variables (e.g. SP_10M) the required components are loaded and @@ -282,8 +280,7 @@ def main(args: Namespace) -> None: if args.baseline_root: init_items = [ - (rt, None) - for rt in iter_baseline_init_times(args.baseline_root, args.step) + (rt, None) for rt in iter_baseline_init_times(args.baseline_root, args.step) ] LOG.info("Found %d baseline init times", len(init_items)) else: @@ -537,8 +534,7 @@ def _strat_da(compute_fn) -> xr.DataArray: n > 0, np.sqrt( np.maximum( - accum_sum_se[(s, h)] / n - - (accum_sum_e[(s, h)] / n) ** 2, + accum_sum_se[(s, h)] / n - (accum_sum_e[(s, h)] / n) ** 2, 0.0, ) ), From 8afc01d20dc512095ef64c9cefe5c03db02ad3af Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Wed, 10 Jun 2026 15:30:47 +0200 Subject: [PATCH 119/129] Port score-maps forecast loading to post-earthkit data_input Swap the removed load_fct_data_from_grib for load_forecast_data; select the target step on the new 'step' (timedelta64) dim instead of 'lead_time'; rename truth coords to latitude/longitude to match the merged map_forecast_to_truth; squeeze the loader's size-1 number/z/forecast_reference_time dims. TOT_PREC de-accumulation now comes from the loader's _tot_prec_handling (our own windowed diff is dropped); we keep requesting only the [step-period, step] window per lead time (memory/disk-bounded per-leadtime design). Baseline (--baseline-root) maps raise NotImplementedError pending a separate port; the positional-IC edge case in _tot_prec_handling is noted inline. --- workflow/scripts/verification_score_maps.py | 88 +++++++++++++-------- 1 file changed, 57 insertions(+), 31 deletions(-) diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index b5b35fd9..e9541180 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -7,6 +7,14 @@ time series in memory. The final BIAS / RMSE / MAE / STDE maps are written to a NetCDF file. +Design note: one Snakemake job per (run, param, lead time), each loading only the +step(s) it needs. We deliberately do not load all lead times at once: per-job +memory and output disk scale with N_leadtimes x grid size, which is infeasible at +interpolator (1 h) and nowcasting (10 min) resolutions; that cost is independent +of GRIB read speed, so it does not improve as loading gets faster. For TOT_PREC +the loader (data_input._tot_prec_handling) de-accumulates over the requested +[step - period, step] window, so we just select the target step. + Usage ----- uv run workflow/scripts/verification_score_maps.py \\ @@ -24,7 +32,7 @@ import numpy as np import xarray as xr -from data_input import load_fct_data_from_grib, load_baseline_from_zarr, parse_steps +from data_input import load_forecast_data, parse_steps from verification.spatial import map_forecast_to_truth LOG = logging.getLogger(__name__) @@ -132,18 +140,21 @@ def _open_zarr_component( da = ds["data"].rename(param).drop_vars("variable", errors="ignore") - # Attach lat/lon as coordinates on the spatial dimension(s). + # Attach latitude/longitude as coordinates on the spatial dimension(s). + # Use the full names to match the forecast loader (load_forecast_data) and + # map_forecast_to_truth, which key on `latitude`/`longitude`. if lat is not None and lon is not None: if spatial_dim is not None: # flat 1-D case: cell/values dim da = da.assign_coords( - lat=(spatial_dim, lat.values), lon=(spatial_dim, lon.values) + latitude=(spatial_dim, lat.values), + longitude=(spatial_dim, lon.values), ) else: # 2-D case: lat/lon still on original flat index — attach via unstack da = da.assign_coords( - lat=(["y", "x"], lat.values.reshape(ny, nx)), - lon=(["y", "x"], lon.values.reshape(ny, nx)), + latitude=(["y", "x"], lat.values.reshape(ny, nx)), + longitude=(["y", "x"], lon.values.reshape(ny, nx)), ) return da @@ -154,7 +165,7 @@ def open_truth_zarr(root: Path, param: str, accum_h: int | None = None) -> xr.Da For derived variables (e.g. SP_10M) the required components are loaded and the derivation is applied on the fly. The returned DataArray has dimensions - ``(time, y, x)`` or ``(time, values)`` and always exposes ``lat``/``lon``. + ``(time, y, x)`` or ``(time, values)`` and always exposes ``latitude``/``longitude``. ``accum_h`` selects the precip accumulation length (TOT_PREC_H). """ if param in _DERIVED: @@ -234,6 +245,18 @@ def main(args: Namespace) -> None: LOG.info("Output : %s", args.output) LOG.info("=" * 60) + if args.baseline_root: + # TODO(port): baseline score maps previously read a per-year + # FCST.zarr archive via data_input.load_baseline_from_zarr, removed in + # the earthkit migration. Baselines now load through the ICON/INCA paths + # (load_forecast_data / load_icon_baseline_from_grib); porting the baseline + # score-maps path onto those is a separate task. Fail fast until then. + raise NotImplementedError( + "Baseline (--baseline-root) score maps are not yet ported to the " + "post-earthkit data_input API (load_baseline_from_zarr was removed). " + "Only run-directory forecasts (--run-root) are supported for now." + ) + # Accumulated params (TOT_PREC) are stored cumulative-from-start, while the # truth is a period accumulation whose length equals the verification step # spacing (e.g. 6h for steps "0/120/6"). Derive that period so we can (a) @@ -340,34 +363,35 @@ def main(args: Namespace) -> None: ) try: - if args.baseline_root: - zarr_path = args.baseline_root / f"FCST{reftime.strftime('%y')}.zarr" - fcst = load_baseline_from_zarr( - root=zarr_path, - reftime=reftime, - steps=req_steps, - params=fct_params, - ) - else: - # For accumulated params (TOT_PREC) req_steps is the - # [step - period, step] window; the loader de-accumulates the - # cumulative-from-start field over those steps, yielding the - # period accumulation at `step`. Instantaneous params request a - # single step. The target step is selected just below. - fcst = load_fct_data_from_grib( - root=grib_dir, - reftime=reftime, - steps=req_steps, - params=fct_params, - ) + # For accumulated params (TOT_PREC) req_steps is the [step - period, + # step] window; load_forecast_data de-accumulates the + # cumulative-from-start field over the requested steps (diff over + # `step`), so the target step holds the period accumulation. + # Instantaneous params request a single step. The target step is + # selected just below. + # + # NOTE: data_input._tot_prec_handling treats the FIRST loaded step + # positionally as the forecast initial condition and zero-fills it + # when it is present-but-all-NaN. For our window the first step is + # `step - period` (non-zero except at the first lead time); a + # corrupt/NaN field there would be silently zeroed, turning the diff + # into a cumulative-from-start value. A missing GRIB instead yields + # NaN and is skipped downstream. The principled fix belongs in + # _tot_prec_handling (gate the IC zero-fill on step 0 being + # requested) — see the port note. + fcst = load_forecast_data(grib_dir, reftime, req_steps, fct_params) except Exception as exc: LOG.warning("Could not load forecast for %s: %s", reftime, exc) n_skip += 1 continue - # Drop lead_time dimension (select only the requested step). - if "lead_time" in fcst.dims: - fcst = fcst.sel(lead_time=np.timedelta64(args.step, "h")) + # Select the target step. The earthkit loader returns forecasts over the + # requested steps with a `step` (timedelta64) dimension; for TOT_PREC the + # loader has already de-accumulated over the window, so the target step + # holds the period accumulation, and for instantaneous params only the + # single requested step is present. + if "step" in fcst.dims: + fcst = fcst.sel(step=np.timedelta64(args.step, "h")) # Compute derived variable if needed. if args.param in _DERIVED: @@ -426,8 +450,10 @@ def main(args: Namespace) -> None: continue fcst_param = fcst_mapped[args.param] - # Squeeze any ensemble/eps dimension (deterministic run stored with size-1 eps). - for dim in ["eps", "ensemble", "number"]: + # Squeeze size-1 non-spatial dims so the error array is purely spatial. + # The earthkit loader keeps `number` (ensemble), `z` (vertical) and + # `forecast_reference_time` as size-1 dims for a deterministic surface run. + for dim in ["eps", "ensemble", "number", "z", "forecast_reference_time"]: if dim in fcst_param.dims and fcst_param.sizes[dim] == 1: fcst_param = fcst_param.squeeze(dim, drop=True) fcst_vals = fcst_param.values From ed56fd4de1421d040713a2abc7b8b132c9458a79 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 11 Jun 2026 11:53:46 +0200 Subject: [PATCH 120/129] Port score-maps baseline loading to post-earthkit data_input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Baselines now load per-reftime through the same load_forecast_data call as runs (routing to the ICON GRIB archive or INCA by root), replacing the removed load_baseline_from_zarr; the zarr-based init-time discovery is dropped — baseline init times come from --reftimes (now required), with missing archive dates skipped at load time. Add --member (control/median/ mean/ID, as in verification_metrics). Guard INCA + accumulated params against step spacings != 1h (INCA delivers native 1h sums that no window-diff applies to); record accum_h and member in the output attrs. Update the verification_score_maps_baseline rule accordingly: stale FCST.zarr inputs replaced by the baseline root, eckit grid files and ECCODES_DEFINITION_PATH added (ICON archive gribs need them), member passed through. Two bugs surfaced by smoke tests against the ICON-CH2-EPS/INCA archives: - load_from_grib_file: coerce Paths to str; earthkit crashes on a single-element list of Path objects, which is exactly what score-maps single-step loads produce (also latent on the run side). - Truth precip units: the script's lazy truth opener missed the m -> mm conversion that MRB-820 put into load_analysis_data_from_zarr, so post-merge TOT_PREC maps would have compared mm forecasts against metre truth. Mirror the conversion in the opener until it is consolidated into data_input (refactor/data-io). TOT_PREC maps are now in true mm (old maps were consistently in m). --- src/data_input/__init__.py | 6 + workflow/rules/verification.smk | 18 +-- workflow/scripts/verification_score_maps.py | 158 +++++++++++--------- 3 files changed, 105 insertions(+), 77 deletions(-) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index 6c14005a..96b3e476 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -180,6 +180,12 @@ def _discover_icon_member_ids( def load_from_grib_file(file: str | list[str], sel_kwargs): + # Coerce Path objects to str: earthkit-data unwraps a single-element list + # into one File source without converting, and then fails on non-str paths. + if isinstance(file, (list, tuple)): + file = [str(f) for f in file] + else: + file = str(file) fieldlist = ekd.from_source("file", file, lazily=True).to_fieldlist() return fieldlist_to_xarray(fieldlist.sel(**sel_kwargs)) diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 6320d296..52032864 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -213,16 +213,12 @@ rule verification_score_maps: rule verification_score_maps_baseline: input: + "src/verification/__init__.py", + "src/data_input/__init__.py", script="workflow/scripts/verification_score_maps.py", - # Declared as inputs purely for dependency tracking (re-run if the baseline - # archive changes). The script discovers the zarrs itself by globbing - # `--baseline_root`, so this list is intentionally not passed on the CLI. - baseline_zarrs=lambda wc: expand( - "{root}/FCST{year}.zarr", - root=BASELINE_CONFIGS[wc.baseline_id].get("root"), - year=sorted({t.strftime("%y") for t in REFTIMES}), - ), + forecast=lambda wc: BASELINE_CONFIGS[wc.baseline_id]["root"], truth=config["truth"]["root"], + eckit_grids=rules.data_download_eckit_geo_grids.output, output: OUT_ROOT / f"data/baselines/{{baseline_id}}/{config['truth']['label']}/score_maps/{{param}}_{{leadtime}}.nc", @@ -234,17 +230,19 @@ rule verification_score_maps_baseline: mem_mb=50_000, runtime="60m", params: - baseline_root=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("root"), baseline_steps=lambda wc: BASELINE_CONFIGS[wc.baseline_id]["steps"], + member=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("member", "000"), reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), shell: """ + export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) uv run {input.script} \ - --baseline_root {params.baseline_root} \ + --baseline_root {input.forecast} \ --reftimes {params.reftimes} \ --truth {input.truth} \ --step {wildcards.leadtime} \ --steps "{params.baseline_steps}" \ --param {wildcards.param} \ + --member "{params.member}" \ --output {output} >{log} 2>&1 """ diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index e9541180..e4496ee8 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -1,11 +1,17 @@ """Compute spatial maps of temporally-aggregated forecast errors. -For a fixed lead time and variable, iterates over all initialisation times found -under a run directory, loads the corresponding GRIB forecast field and the -matching truth slice from a reference zarr, maps the forecast onto the truth -grid, and accumulates running error statistics without ever holding the full -time series in memory. The final BIAS / RMSE / MAE / STDE maps are written to a -NetCDF file. +For a fixed lead time and variable, iterates over all initialisation times +(discovered under a run directory, or taken from --reftimes for baselines), +loads the corresponding forecast field and the matching truth slice from a +reference zarr, maps the forecast onto the truth grid, and accumulates running +error statistics without ever holding the full time series in memory. The +final BIAS / RMSE / MAE / STDE maps are written to a NetCDF file. + +Forecasts load through data_input.load_forecast_data, which routes by source: +ML run directories (GRIB files), INCA (NetCDF archive), or otherwise the ICON +operational GRIB archive. Baselines (--baseline_root) use the latter two paths; +init times are not discovered from the archive but taken from --reftimes, with +unavailable dates skipped at load time. Design note: one Snakemake job per (run, param, lead time), each loading only the step(s) it needs. We deliberately do not load all lead times at once: per-job @@ -104,13 +110,15 @@ def _compute_derived(ds: xr.Dataset, param: str) -> xr.DataArray: # --------------------------------------------------------------------------- # Truth loading # --------------------------------------------------------------------------- -# TODO: consolidate with src/data_input/__init__.py once the ongoing -# data-loading refactor lands. _open_zarr_component below duplicates +# TODO: consolidate with src/data_input/__init__.py as part of the +# refactor/data-io branch. _open_zarr_component below duplicates # ~80% of load_analysis_data_from_zarr but returns a lazy DataArray # rather than a time-sliced Dataset, which is what our streaming # aggregation needs. The right end-state is a shared lazy-open primitive -# in data_input that both consumers use; not introduced here because -# data_input is being reworked separately and we don't want to conflict. +# in data_input that both consumers use; not introduced here to avoid +# conflicting with the data-io refactor. Until then this opener must +# mirror the loader's conventions (notably the m -> mm precip conversion +# from MRB-820). def _open_zarr_component( @@ -140,6 +148,13 @@ def _open_zarr_component( da = ds["data"].rename(param).drop_vars("variable", errors="ignore") + # Truth zarrs store precip in m (anemoi convention); all forecast loaders + # deliver canonical mm (kg m-2) since MRB-820, which put this conversion in + # load_analysis_data_from_zarr. Mirror it here until this opener is + # consolidated into data_input (refactor/data-io). Stays lazy (dask). + if param in _ACCUMULATED_PARAMS: + da = da * 1000 + # Attach latitude/longitude as coordinates on the spatial dimension(s). # Use the full names to match the forecast loader (load_forecast_data) and # map_forecast_to_truth, which key on `latitude`/`longitude`. @@ -208,30 +223,6 @@ def iter_init_dirs(run_root: Path) -> list[tuple[datetime, Path]]: return result -def iter_baseline_init_times(baseline_root: Path, step: int) -> list[datetime]: - """Return all init times from a baseline's zarr(s) that have the requested step available. - - The per-year zarrs are discovered by globbing ``FCST*.zarr`` under ``baseline_root`` - rather than taking an explicit list: the layout is fixed and the discovered init - times are filtered down to the configured reftimes by the caller anyway, so any - extra years picked up from the archive are harmless. - """ - step_td = np.timedelta64(step, "h") - reftimes = [] - for zarr_path in sorted(baseline_root.glob("FCST*.zarr")): - if not zarr_path.exists(): - LOG.warning("Baseline zarr not found: %s", zarr_path) - continue - ds = xr.open_zarr(zarr_path, consolidated=True, decode_timedelta=True) - if step_td not in ds["step"].values: - LOG.warning("Step %dh not in %s, skipping", step, zarr_path) - continue - for rt in ds["forecast_reference_time"].values: - ts = (rt - np.datetime64("1970-01-01T00:00:00")) / np.timedelta64(1, "s") - reftimes.append(datetime.utcfromtimestamp(float(ts))) - return sorted(reftimes) - - # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -245,18 +236,6 @@ def main(args: Namespace) -> None: LOG.info("Output : %s", args.output) LOG.info("=" * 60) - if args.baseline_root: - # TODO(port): baseline score maps previously read a per-year - # FCST.zarr archive via data_input.load_baseline_from_zarr, removed in - # the earthkit migration. Baselines now load through the ICON/INCA paths - # (load_forecast_data / load_icon_baseline_from_grib); porting the baseline - # score-maps path onto those is a separate task. Fail fast until then. - raise NotImplementedError( - "Baseline (--baseline-root) score maps are not yet ported to the " - "post-earthkit data_input API (load_baseline_from_zarr was removed). " - "Only run-directory forecasts (--run-root) are supported for now." - ) - # Accumulated params (TOT_PREC) are stored cumulative-from-start, while the # truth is a period accumulation whose length equals the verification step # spacing (e.g. 6h for steps "0/120/6"). Derive that period so we can (a) @@ -284,6 +263,19 @@ def main(args: Namespace) -> None: ) req_steps = [args.step - accum_h, args.step] LOG.info("Accumulation period: %dh (forecast window %s)", accum_h, req_steps) + + # INCA delivers native 1h precip sums and (unlike the GRIB paths, where + # the cumulative-from-start diff adapts to the requested window) cannot + # re-aggregate to a coarser period: the value at the target step would + # stay a 1h sum while the truth read is TOT_PREC_H — a silent + # mismatch. Re-aggregation in the loader is a planned follow-up. + if args.baseline_root and "INCA" in args.baseline_root.parts and accum_h != 1: + raise ValueError( + f"INCA provides native 1h accumulations only, but the step " + f"spacing of --steps '{args.steps}' implies a {accum_h}h " + f"accumulation period for '{args.param}'. Use 1h-spaced steps " + f"for INCA score maps." + ) else: req_steps = [args.step] @@ -302,22 +294,23 @@ def main(args: Namespace) -> None: LOG.info("Truth opened lazily: %s", truth_da) if args.baseline_root: + # The operational archive is too large to enumerate up front; the + # experiment's configured init times define the work list, and dates + # missing from the archive are skipped at load time below. init_items = [ - (rt, None) for rt in iter_baseline_init_times(args.baseline_root, args.step) + (rt, None) + for rt in sorted(datetime.strptime(s, DATETIME_FMT) for s in args.reftimes) ] - LOG.info("Found %d baseline init times", len(init_items)) + LOG.info("Using %d baseline init times from --reftimes", len(init_items)) else: init_items = iter_init_dirs(args.run_root) LOG.info("Found %d init time directories", len(init_items)) - # Restrict to the experiment's configured init times if provided. - # Without this, baseline zarrs (which contain a continuous archive) would - # cause the script to process every init time in the file rather than - # only those in the user's hindcast period. - if args.reftimes: - wanted = {datetime.strptime(s, DATETIME_FMT) for s in args.reftimes} - init_items = [(rt, d) for rt, d in init_items if rt in wanted] - LOG.info("Filtered to %d init times matching --reftimes", len(init_items)) + # Restrict to the experiment's configured init times if provided. + if args.reftimes: + wanted = {datetime.strptime(s, DATETIME_FMT) for s in args.reftimes} + init_items = [(rt, d) for rt, d in init_items if rt in wanted] + LOG.info("Filtered to %d init times matching --reftimes", len(init_items)) step_td = timedelta(hours=args.step) @@ -364,11 +357,13 @@ def main(args: Namespace) -> None: try: # For accumulated params (TOT_PREC) req_steps is the [step - period, - # step] window; load_forecast_data de-accumulates the - # cumulative-from-start field over the requested steps (diff over - # `step`), so the target step holds the period accumulation. - # Instantaneous params request a single step. The target step is - # selected just below. + # step] window; for GRIB sources (runs and the ICON archive) the + # loader de-accumulates the cumulative-from-start field over the + # requested steps (diff over `step`), so the target step holds the + # period accumulation; INCA returns native 1h sums, matching the + # period because accum_h == 1 is enforced above. Instantaneous + # params request a single step. The target step is selected just + # below. # # NOTE: data_input._tot_prec_handling treats the FIRST loaded step # positionally as the forecast initial condition and zero-fills it @@ -378,8 +373,12 @@ def main(args: Namespace) -> None: # into a cumulative-from-start value. A missing GRIB instead yields # NaN and is skipped downstream. The principled fix belongs in # _tot_prec_handling (gate the IC zero-fill on step 0 being - # requested) — see the port note. - fcst = load_forecast_data(grib_dir, reftime, req_steps, fct_params) + # requested) — see the port note. This applies to runs and ICON + # baselines alike. + src_root = args.baseline_root if args.baseline_root else grib_dir + fcst = load_forecast_data( + src_root, reftime, req_steps, fct_params, member=args.member + ) except Exception as exc: LOG.warning("Could not load forecast for %s: %s", reftime, exc) n_skip += 1 @@ -572,6 +571,10 @@ def _strat_da(compute_fn) -> xr.DataArray: attrs={ "param": args.param, "step_h": args.step, + # Accumulation period of the verified quantity (accumulated params + # only) — lets consumers tell a 1h INCA map from a 6h ICON map. + "accum_h": accum_h if accum_h is not None else "n/a", + "member": args.member, "source": str(args.baseline_root if args.baseline_root else args.run_root), "n_processed": n_ok, "n_skipped": n_skip, @@ -588,7 +591,8 @@ def _strat_da(compute_fn) -> xr.DataArray: parser = ArgumentParser( description=( "Compute spatial maps of temporally-aggregated forecast errors. " - "Supports both model runs (GRIB) and baselines (zarr). " + "Supports model runs (GRIB) and baselines (ICON GRIB archive or " + "INCA NetCDF archive). " "Exactly one of --run_root or --baseline_root must be provided." ) ) @@ -602,7 +606,21 @@ def _strat_da(compute_fn) -> xr.DataArray: "--baseline_root", type=Path, default=None, - help="Root directory of a baseline (e.g. /path/to/ICON-CH1-EPS), containing FCST.zarr files.", + help=( + "Root directory of a baseline archive (e.g. the ICON-CH1/CH2-EPS " + "operational GRIB archive, or an INCA NetCDF archive). Requires " + "--reftimes." + ), + ) + parser.add_argument( + "--member", + type=str, + default="000", + help=( + "Ensemble member to load for ICON baselines: '000' for control, " + "'median' for the pre-computed median, 'mean' to average all " + "members, or any 3-digit member ID. Ignored for runs and INCA." + ), ) parser.add_argument( "--truth", @@ -639,8 +657,9 @@ def _strat_da(compute_fn) -> xr.DataArray: nargs="+", default=None, help=( - "Optional list of init times (YYYYMMDDHHMM) to restrict processing to. " - "Required for baselines whose zarr contains a continuous archive." + "List of init times (YYYYMMDDHHMM). For runs: optional restriction of " + "the discovered init-time directories. For baselines: required; " + "defines the init times to load from the archive." ), ) parser.add_argument( @@ -653,5 +672,10 @@ def _strat_da(compute_fn) -> xr.DataArray: if bool(args.run_root) == bool(args.baseline_root): parser.error("Exactly one of --run_root or --baseline_root must be provided.") + if args.baseline_root and not args.reftimes: + parser.error( + "--reftimes is required with --baseline_root: init times cannot be " + "discovered from the operational archive." + ) main(args) From 4d8fe7e076efd3341e753233f918a12957883bac Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Thu, 11 Jun 2026 14:46:42 +0200 Subject: [PATCH 121/129] Move score_maps config under experiment section Score maps are experiment outputs (triggered via evalml experiment --maps), so their config belongs under experiment:, consistent with scorecards. Move the ScoreMapsConfig field from ConfigModel to ExperimentConfig, update the Snakefile lookups, re-nest the block in the example configs (adding it to temporal-downscalers-ich1.yaml), and regenerate the JSON schema. Addresses review comment by @dnerini on MRB-650. --- config/forecasters-ich1-oper-fixed.yaml | 65 ++++++++++++------------- config/forecasters-ich1-oper.yaml | 65 ++++++++++++------------- config/forecasters-ich1.yaml | 65 ++++++++++++------------- config/temporal-downscalers-ich1.yaml | 32 ++++++++++++ src/evalml/config.py | 8 +-- workflow/Snakefile | 27 +++++----- workflow/tools/config.schema.json | 8 +-- 7 files changed, 151 insertions(+), 119 deletions(-) diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index 799be5c1..aaa87a5c 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -70,6 +70,38 @@ experiment: # - init_hour # - region - season + score_maps: + params: + - T_2M + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" + scores: + - BIAS + # - RMSE + # - MAE + regions: + - switzerland + # - centraleurope + seasons: + - all + # - DJF + # - MAM + # - JJA + # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" locations: output_root: output/ @@ -85,36 +117,3 @@ profile: runtime: "1h" gpus: 0 jobs: 50 - -score_maps: - params: - - T_2M - # - TD_2M - # - U_10M - # - V_10M - # - SP_10M - # - PS - # - PMSL - # - TOT_PREC - leadtimes: [6, 24] - # Or, to compute every available leadtime from runs+baselines: - # leadtimes: "all" - scores: - - BIAS - # - RMSE - # - MAE - regions: - - switzerland - # - centraleurope - seasons: - - all - # - DJF - # - MAM - # - JJA - # - SON - init_hours: - - all - # - "00" - # - "06" - # - "12" - # - "18" diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 95648b60..116df4d4 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -67,6 +67,38 @@ experiment: # - init_hour # - region - season + score_maps: + params: + - T_2M + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" + scores: + - BIAS + # - RMSE + # - MAE + regions: + - switzerland + # - centraleurope + seasons: + - all + # - DJF + # - MAM + # - JJA + # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" locations: output_root: output/ @@ -82,36 +114,3 @@ profile: runtime: "1h" gpus: 0 jobs: 50 - -score_maps: - params: - - T_2M - # - TD_2M - # - U_10M - # - V_10M - # - SP_10M - # - PS - # - PMSL - # - TOT_PREC - leadtimes: [6, 24] - # Or, to compute every available leadtime from runs+baselines: - # leadtimes: "all" - scores: - - BIAS - # - RMSE - # - MAE - regions: - - switzerland - # - centraleurope - seasons: - - all - # - DJF - # - MAM - # - JJA - # - SON - init_hours: - - all - # - "00" - # - "06" - # - "12" - # - "18" diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index 9f023e32..1f62f6e0 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -79,6 +79,38 @@ experiment: # - init_hour # - region - season + score_maps: + params: + - T_2M + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" + scores: + - BIAS + # - RMSE + # - MAE + regions: + - switzerland + # - centraleurope + seasons: + - all + # - DJF + # - MAM + # - JJA + # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" locations: output_root: output/ @@ -94,36 +126,3 @@ profile: runtime: "1h" gpus: 0 jobs: 50 - -score_maps: - params: - - T_2M - # - TD_2M - # - U_10M - # - V_10M - # - SP_10M - # - PS - # - PMSL - # - TOT_PREC - leadtimes: [6, 24] - # Or, to compute every available leadtime from runs+baselines: - # leadtimes: "all" - scores: - - BIAS - # - RMSE - # - MAE - regions: - - switzerland - # - centraleurope - seasons: - - all - # - DJF - # - MAM - # - JJA - # - SON - init_hours: - - all - # - "00" - # - "06" - # - "12" - # - "18" diff --git a/config/temporal-downscalers-ich1.yaml b/config/temporal-downscalers-ich1.yaml index 51ec4ed9..5258c698 100644 --- a/config/temporal-downscalers-ich1.yaml +++ b/config/temporal-downscalers-ich1.yaml @@ -98,6 +98,38 @@ experiment: - "V_10M:RMSE,R2,ETS" - "T_2M:RMSE,R2,ETS" - "TOT_PREC:RMSE,R2,ETS" + score_maps: + params: + - T_2M + # - TD_2M + # - U_10M + # - V_10M + # - SP_10M + # - PS + # - PMSL + # - TOT_PREC + leadtimes: [6, 24] + # Or, to compute every available leadtime from runs+baselines: + # leadtimes: "all" + scores: + - BIAS + # - RMSE + # - MAE + regions: + - switzerland + # - centraleurope + seasons: + - all + # - DJF + # - MAM + # - JJA + # - SON + init_hours: + - all + # - "00" + # - "06" + # - "12" + # - "18" showcase: params: diff --git a/src/evalml/config.py b/src/evalml/config.py index 1d6c041c..5fee7cd7 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -419,6 +419,10 @@ class ExperimentConfig(BaseModel): default=None, description="Scorecard generation configuration. Omit or set enabled: false to disable.", ) + score_maps: ScoreMapsConfig = Field( + default_factory=ScoreMapsConfig, + description="Parameters for score map plots (used with --maps flag).", + ) @field_validator("thresholds") @classmethod @@ -526,10 +530,6 @@ class ConfigModel(BaseModel): ) locations: Locations profile: Profile - score_maps: ScoreMapsConfig = Field( - default_factory=ScoreMapsConfig, - description="Parameters for score map plots (used with --maps flag).", - ) showcase: ShowcaseConfig = Field( default_factory=ShowcaseConfig, description="Settings for the showcase workflow.", diff --git a/workflow/Snakefile b/workflow/Snakefile index cb5e5a5c..1e4379e7 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -160,29 +160,32 @@ rule experiment_all: ), +SCORE_MAPS_CONFIG = config["experiment"]["score_maps"] + + rule score_maps_all: """Target rule for score maps (opt-in via evalml experiment --maps).""" input: expand( rules.plot_score_maps.output, run_id=collect_all_candidates(), - leadtime=resolve_leadtimes(config["score_maps"]["leadtimes"]), - score=config["score_maps"]["scores"], - param=config["score_maps"]["params"], - region=config["score_maps"]["regions"], - season=config["score_maps"]["seasons"], - init_hour=config["score_maps"]["init_hours"], + leadtime=resolve_leadtimes(SCORE_MAPS_CONFIG["leadtimes"]), + score=SCORE_MAPS_CONFIG["scores"], + param=SCORE_MAPS_CONFIG["params"], + region=SCORE_MAPS_CONFIG["regions"], + season=SCORE_MAPS_CONFIG["seasons"], + init_hour=SCORE_MAPS_CONFIG["init_hours"], experiment=EXPERIMENT_NAME, ), expand( rules.plot_score_maps_baseline.output, baseline_id=list(BASELINE_CONFIGS), - leadtime=resolve_leadtimes(config["score_maps"]["leadtimes"]), - score=config["score_maps"]["scores"], - param=config["score_maps"]["params"], - region=config["score_maps"]["regions"], - season=config["score_maps"]["seasons"], - init_hour=config["score_maps"]["init_hours"], + leadtime=resolve_leadtimes(SCORE_MAPS_CONFIG["leadtimes"]), + score=SCORE_MAPS_CONFIG["scores"], + param=SCORE_MAPS_CONFIG["params"], + region=SCORE_MAPS_CONFIG["regions"], + season=SCORE_MAPS_CONFIG["seasons"], + init_hour=SCORE_MAPS_CONFIG["init_hours"], experiment=EXPERIMENT_NAME, ), diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index 4f2cde04..d5d2cadd 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -261,6 +261,10 @@ ], "default": null, "description": "Scorecard generation configuration. Omit or set enabled: false to disable." + }, + "score_maps": { + "$ref": "#/$defs/ScoreMapsConfig", + "description": "Parameters for score map plots (used with --maps flag)." } }, "required": [ @@ -958,10 +962,6 @@ "profile": { "$ref": "#/$defs/Profile" }, - "score_maps": { - "$ref": "#/$defs/ScoreMapsConfig", - "description": "Parameters for score map plots (used with --maps flag)." - }, "showcase": { "$ref": "#/$defs/ShowcaseConfig", "description": "Settings for the showcase workflow." From c941695caa5427aa77ec8ed8452807baaec909c0 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Fri, 12 Jun 2026 16:30:31 +0200 Subject: [PATCH 122/129] Replace --maps CLI flag with score_maps.enabled config option Address review feedback: score maps are now toggled via an "enabled" field in the experiment.score_maps config block, consistent with meteograms, animations and scorecards. - Add enabled field (default false) to ScoreMapsConfig - Slim score_maps defaults to a cheap starter set (T_2M, BIAS, leadtimes 6/24 h, switzerland) so enabling alone stays inexpensive - Include score_maps_all input in experiment_all when enabled; keep score_maps_all as a direct Snakemake target - Remove --maps flag and extra_targets plumbing from the CLI - Reduce example configs to a minimal "enabled: false" block --- config/forecasters-ich1-oper-fixed.yaml | 32 +------------ config/forecasters-ich1-oper.yaml | 32 +------------ config/forecasters-ich1.yaml | 32 +------------ config/temporal-downscalers-ich1.yaml | 32 +------------ src/evalml/cli.py | 11 +---- src/evalml/config.py | 22 +++++---- workflow/Snakefile | 61 +++++++++++++------------ workflow/tools/config.schema.json | 56 ++++++----------------- 8 files changed, 64 insertions(+), 214 deletions(-) diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index aaa87a5c..1c8c2519 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -71,37 +71,7 @@ experiment: # - region - season score_maps: - params: - - T_2M - # - TD_2M - # - U_10M - # - V_10M - # - SP_10M - # - PS - # - PMSL - # - TOT_PREC - leadtimes: [6, 24] - # Or, to compute every available leadtime from runs+baselines: - # leadtimes: "all" - scores: - - BIAS - # - RMSE - # - MAE - regions: - - switzerland - # - centraleurope - seasons: - - all - # - DJF - # - MAM - # - JJA - # - SON - init_hours: - - all - # - "00" - # - "06" - # - "12" - # - "18" + enabled: false locations: output_root: output/ diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 116df4d4..b860f434 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -68,37 +68,7 @@ experiment: # - region - season score_maps: - params: - - T_2M - # - TD_2M - # - U_10M - # - V_10M - # - SP_10M - # - PS - # - PMSL - # - TOT_PREC - leadtimes: [6, 24] - # Or, to compute every available leadtime from runs+baselines: - # leadtimes: "all" - scores: - - BIAS - # - RMSE - # - MAE - regions: - - switzerland - # - centraleurope - seasons: - - all - # - DJF - # - MAM - # - JJA - # - SON - init_hours: - - all - # - "00" - # - "06" - # - "12" - # - "18" + enabled: false locations: output_root: output/ diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index 1f62f6e0..576ce764 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -80,37 +80,7 @@ experiment: # - region - season score_maps: - params: - - T_2M - # - TD_2M - # - U_10M - # - V_10M - # - SP_10M - # - PS - # - PMSL - # - TOT_PREC - leadtimes: [6, 24] - # Or, to compute every available leadtime from runs+baselines: - # leadtimes: "all" - scores: - - BIAS - # - RMSE - # - MAE - regions: - - switzerland - # - centraleurope - seasons: - - all - # - DJF - # - MAM - # - JJA - # - SON - init_hours: - - all - # - "00" - # - "06" - # - "12" - # - "18" + enabled: false locations: output_root: output/ diff --git a/config/temporal-downscalers-ich1.yaml b/config/temporal-downscalers-ich1.yaml index 5258c698..21dfd5ec 100644 --- a/config/temporal-downscalers-ich1.yaml +++ b/config/temporal-downscalers-ich1.yaml @@ -99,37 +99,7 @@ experiment: - "T_2M:RMSE,R2,ETS" - "TOT_PREC:RMSE,R2,ETS" score_maps: - params: - - T_2M - # - TD_2M - # - U_10M - # - V_10M - # - SP_10M - # - PS - # - PMSL - # - TOT_PREC - leadtimes: [6, 24] - # Or, to compute every available leadtime from runs+baselines: - # leadtimes: "all" - scores: - - BIAS - # - RMSE - # - MAE - regions: - - switzerland - # - centraleurope - seasons: - - all - # - DJF - # - MAM - # - JJA - # - SON - init_hours: - - all - # - "00" - # - "06" - # - "12" - # - "18" + enabled: false showcase: params: diff --git a/src/evalml/cli.py b/src/evalml/cli.py index fcc295ff..f3df3bf4 100644 --- a/src/evalml/cli.py +++ b/src/evalml/cli.py @@ -129,7 +129,6 @@ def execute_workflow( dag: bool = False, rulegraph: bool = False, extra_smk_args: tuple[str, ...] = (), - extra_targets: list[str] = [], ): if dag or rulegraph: generate_graph( @@ -147,7 +146,7 @@ def execute_workflow( if report and not dry_run: command += ["--report-after-run", "--report", str(report)] - command += [target] + extra_targets + command += [target] command += list(extra_smk_args) if not verbose: command += ["--quiet", "rules"] # reduce verobosity of snakemake output @@ -164,16 +163,9 @@ def cli(): @click.argument( "configfile", type=click.Path(exists=True, dir_okay=False, path_type=Path) ) -@click.option( - "--maps", - is_flag=True, - default=False, - help="Also produce score maps (computationally intensive).", -) @workflow_options def experiment( configfile, - maps, cores, verbose, dry_run, @@ -194,7 +186,6 @@ def experiment( dag, rulegraph, extra_smk_args, - extra_targets=["score_maps_all"] if maps else [], ) diff --git a/src/evalml/config.py b/src/evalml/config.py index 5fee7cd7..b5b42402 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -222,15 +222,19 @@ class BaselineItem(BaseModel): class ScoreMapsConfig(BaseModel): """Parameters controlling which score map plots are produced.""" + enabled: bool = Field( + default=False, + description="Whether to produce score maps (computationally intensive).", + ) params: List[str] = Field( - default=["T_2M", "TD_2M", "U_10M", "V_10M", "SP_10M", "PS", "PMSL", "TOT_PREC"], + default=["T_2M"], description=( "List of parameters to plot. Supported values: T_2M, TD_2M, U_10M, V_10M, " "PS, PMSL, TOT_PREC (native), and SP_10M (derived wind speed from U_10M/V_10M)." ), ) leadtimes: List[int] | Literal["all"] = Field( - default=list(range(6, 121, 6)), + default=[6, 24], description=( "List of lead times (hours) to plot, or the literal string 'all' " "to expand to the union of step lists from all configured runs " @@ -238,16 +242,16 @@ class ScoreMapsConfig(BaseModel): ), ) scores: List[str] = Field( - default=["BIAS", "RMSE", "MAE"], - description="List of verification scores to plot.", + default=["BIAS"], + description="List of verification scores to plot. Supported: BIAS, RMSE, MAE.", ) regions: List[str] = Field( - default=["switzerland", "centraleurope"], - description="List of regions to plot.", + default=["switzerland"], + description="List of regions to plot (e.g. switzerland, centraleurope).", ) seasons: List[str] = Field( - default=["all", "DJF", "MAM", "JJA", "SON"], - description="List of seasons to plot.", + default=["all"], + description="List of seasons to plot ('all', 'DJF', 'MAM', 'JJA', 'SON').", ) init_hours: List[str] = Field( default=["all"], @@ -421,7 +425,7 @@ class ExperimentConfig(BaseModel): ) score_maps: ScoreMapsConfig = Field( default_factory=ScoreMapsConfig, - description="Parameters for score map plots (used with --maps flag).", + description="Score map plot configuration. Set enabled: true to produce score maps.", ) @field_validator("thresholds") diff --git a/workflow/Snakefile b/workflow/Snakefile index 1e4379e7..24214458 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -130,6 +130,36 @@ onerror: # ----------------------------------------------------- +SCORE_MAPS_CONFIG = config["experiment"]["score_maps"] + + +rule score_maps_all: + """Target rule for score maps (opt-in via experiment.score_maps.enabled).""" + input: + expand( + rules.plot_score_maps.output, + run_id=collect_all_candidates(), + leadtime=resolve_leadtimes(SCORE_MAPS_CONFIG["leadtimes"]), + score=SCORE_MAPS_CONFIG["scores"], + param=SCORE_MAPS_CONFIG["params"], + region=SCORE_MAPS_CONFIG["regions"], + season=SCORE_MAPS_CONFIG["seasons"], + init_hour=SCORE_MAPS_CONFIG["init_hours"], + experiment=EXPERIMENT_NAME, + ), + expand( + rules.plot_score_maps_baseline.output, + baseline_id=list(BASELINE_CONFIGS), + leadtime=resolve_leadtimes(SCORE_MAPS_CONFIG["leadtimes"]), + score=SCORE_MAPS_CONFIG["scores"], + param=SCORE_MAPS_CONFIG["params"], + region=SCORE_MAPS_CONFIG["regions"], + season=SCORE_MAPS_CONFIG["seasons"], + init_hour=SCORE_MAPS_CONFIG["init_hours"], + experiment=EXPERIMENT_NAME, + ), + + rule experiment_all: """Target rule for experiment workflow.""" input: @@ -158,36 +188,7 @@ rule experiment_all: ) else [] ), - - -SCORE_MAPS_CONFIG = config["experiment"]["score_maps"] - - -rule score_maps_all: - """Target rule for score maps (opt-in via evalml experiment --maps).""" - input: - expand( - rules.plot_score_maps.output, - run_id=collect_all_candidates(), - leadtime=resolve_leadtimes(SCORE_MAPS_CONFIG["leadtimes"]), - score=SCORE_MAPS_CONFIG["scores"], - param=SCORE_MAPS_CONFIG["params"], - region=SCORE_MAPS_CONFIG["regions"], - season=SCORE_MAPS_CONFIG["seasons"], - init_hour=SCORE_MAPS_CONFIG["init_hours"], - experiment=EXPERIMENT_NAME, - ), - expand( - rules.plot_score_maps_baseline.output, - baseline_id=list(BASELINE_CONFIGS), - leadtime=resolve_leadtimes(SCORE_MAPS_CONFIG["leadtimes"]), - score=SCORE_MAPS_CONFIG["scores"], - param=SCORE_MAPS_CONFIG["params"], - region=SCORE_MAPS_CONFIG["regions"], - season=SCORE_MAPS_CONFIG["seasons"], - init_hour=SCORE_MAPS_CONFIG["init_hours"], - experiment=EXPERIMENT_NAME, - ), + (rules.score_maps_all.input if SCORE_MAPS_CONFIG["enabled"] else []), rule showcase_all: diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index d5d2cadd..8e677694 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -264,7 +264,7 @@ }, "score_maps": { "$ref": "#/$defs/ScoreMapsConfig", - "description": "Parameters for score map plots (used with --maps flag)." + "description": "Score map plot configuration. Set enabled: true to produce score maps." } }, "required": [ @@ -573,16 +573,15 @@ "ScoreMapsConfig": { "description": "Parameters controlling which score map plots are produced.", "properties": { + "enabled": { + "default": false, + "description": "Whether to produce score maps (computationally intensive).", + "title": "Enabled", + "type": "boolean" + }, "params": { "default": [ - "T_2M", - "TD_2M", - "U_10M", - "V_10M", - "SP_10M", - "PS", - "PMSL", - "TOT_PREC" + "T_2M" ], "description": "List of parameters to plot. Supported values: T_2M, TD_2M, U_10M, V_10M, PS, PMSL, TOT_PREC (native), and SP_10M (derived wind speed from U_10M/V_10M).", "items": { @@ -606,36 +605,16 @@ ], "default": [ 6, - 12, - 18, - 24, - 30, - 36, - 42, - 48, - 54, - 60, - 66, - 72, - 78, - 84, - 90, - 96, - 102, - 108, - 114, - 120 + 24 ], "description": "List of lead times (hours) to plot, or the literal string 'all' to expand to the union of step lists from all configured runs and baselines.", "title": "Leadtimes" }, "scores": { "default": [ - "BIAS", - "RMSE", - "MAE" + "BIAS" ], - "description": "List of verification scores to plot.", + "description": "List of verification scores to plot. Supported: BIAS, RMSE, MAE.", "items": { "type": "string" }, @@ -644,10 +623,9 @@ }, "regions": { "default": [ - "switzerland", - "centraleurope" + "switzerland" ], - "description": "List of regions to plot.", + "description": "List of regions to plot (e.g. switzerland, centraleurope).", "items": { "type": "string" }, @@ -656,13 +634,9 @@ }, "seasons": { "default": [ - "all", - "DJF", - "MAM", - "JJA", - "SON" + "all" ], - "description": "List of seasons to plot.", + "description": "List of seasons to plot ('all', 'DJF', 'MAM', 'JJA', 'SON').", "items": { "type": "string" }, From af061b922b13bf27561b0a8784659679b1b368bd Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 15 Jun 2026 10:14:48 +0200 Subject: [PATCH 123/129] Fix TOT_PREC step-0 handling for first-lead-time accumulation windows anemoi-inference omits the TOT_PREC field at step 0, so the [0, period] window needed for first-lead-time score maps crashed (single step, empty diff) and full-range loads silently returned all-NaN precip at the first lead time. Gate the zero-IC synthesis on step 0 being requested: synthesise a zero initial condition when step 0 is requested but absent or all-NaN, leave windows that do not include step 0 untouched, and raise clearly when a lone step cannot form an accumulation window. Thread the requested steps through load_forecast_data_from_grib to all callers. Update the score-maps script comment to the new contract and add unit tests covering each branch. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/data_input/__init__.py | 57 ++++++++++++++++++--- tests/unit/test_data_input.py | 55 ++++++++++++++++++++ workflow/scripts/verification_score_maps.py | 15 ++---- 3 files changed, 110 insertions(+), 17 deletions(-) create mode 100644 tests/unit/test_data_input.py diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index 96b3e476..402585e7 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -218,13 +218,40 @@ def fieldlist_to_xarray(fieldlist) -> xr.Dataset: return ds -def _tot_prec_handling(tp: xr.DataArray) -> xr.DataArray: +def _tot_prec_handling( + tp: xr.DataArray, requested_steps: list[int] | None = None +) -> xr.DataArray: _full_step_coord = tp["step"] # step coordinate before .diff() # anemoi-inference sometimes omits step 0 from the GRIB even with - # accumulate_from_start_of_forecast enabled. If missing, earthkit-data - # will fill it with NaNs following the `allow_holes=True` flag. - if tp[{"step": 0}].isnull().all(): + # accumulate_from_start_of_forecast enabled: the field may be absent from + # the step coordinate entirely, or present but NaN-filled by earthkit-data + # (allow_holes=True). With cumulative-from-start data the accumulation at + # the initial condition is identically zero, so synthesise it — but only + # when step 0 was actually requested (`requested_steps`); for window loads + # like [18, 24] the first step is real data and must not be treated as an + # initial condition. + if requested_steps is not None: + if 0 in requested_steps: + step0_idx = np.where(tp["step"].values == np.timedelta64(0, "ns"))[0] + if step0_idx.size == 0: + LOG.warning( + "Step 0 of TOT_PREC is missing from the GRIB, prepending " + "zeroes assuming accumulate_from_start_of_forecast is " + "enabled." + ) + zero = xr.zeros_like(tp.isel(step=[0])) + zero = zero.assign_coords(step=[np.timedelta64(0, "ns")]) + tp = xr.concat([zero, tp], dim="step") + elif tp[{"step": int(step0_idx[0])}].isnull().all(): + LOG.warning( + "Step 0 of TOT_PREC is all-NaN, filling with zeroes " + "assuming accumulate_from_start_of_forecast is enabled." + ) + tp[{"step": int(step0_idx[0])}] = 0.0 + elif tp[{"step": 0}].isnull().all(): + # Legacy path for callers that do not pass the requested steps: treat + # the first loaded step positionally as the initial condition. LOG.warning( "Step 0 of TOT_PREC is missing, filling with zeroes " "assuming accumulate_from_start_of_forecast is enabled." @@ -238,6 +265,12 @@ def _tot_prec_handling(tp: xr.DataArray) -> xr.DataArray: "Disaggregating TOT_PREC from cumulative-from-start to per-step accumulations." ) tp = tp.diff("step") + if tp.sizes["step"] == 0: + raise ValueError( + "Cannot de-accumulate TOT_PREC: only a single step was loaded and " + "step 0 was not requested/synthesised, so no accumulation window " + "can be formed. Request the preceding step as well." + ) # Sanity-check that the incoming data is actually cumulative. If # some values are significantly negative, it indicates that the data @@ -260,12 +293,19 @@ def _tot_prec_handling(tp: xr.DataArray) -> xr.DataArray: return tp -def load_forecast_data_from_grib(files: list[Path], params: list[str]) -> xr.Dataset: - """Load forecast data from a list of GRIB files.""" +def load_forecast_data_from_grib( + files: list[Path], params: list[str], steps: list[int] | None = None +) -> xr.Dataset: + """Load forecast data from a list of GRIB files. + + `steps` (lead times in hours, if known) is forwarded to the TOT_PREC + de-accumulation so a missing step-0 field can be synthesised as zero when + step 0 was requested. + """ ds = load_from_grib_file(files, {"parameter.variable": params}) if "TOT_PREC" in ds.data_vars: - ds["TOT_PREC"] = _tot_prec_handling(ds["TOT_PREC"]) + ds["TOT_PREC"] = _tot_prec_handling(ds["TOT_PREC"], requested_steps=steps) return ds @@ -719,6 +759,7 @@ def load_icon_baseline_from_grib( root, reftime, steps, member_id=mid ), params=params, + steps=steps, ) if "number" in ds.dims: ds = ds.isel(number=0, drop=True) @@ -736,6 +777,7 @@ def load_icon_baseline_from_grib( return load_forecast_data_from_grib( files=_collect_icon_archive_files(root, reftime, steps, member_id=member), params=params, + steps=steps, ) @@ -757,6 +799,7 @@ def load_forecast_data( # NOTE: root is already for a specific reftime files=_collect_ml_grib_files(root, steps), params=params, + steps=steps, ) if "INCA" in root.parts: LOG.info("Loading INCA baseline from NetCDF files...") diff --git a/tests/unit/test_data_input.py b/tests/unit/test_data_input.py new file mode 100644 index 00000000..912bad68 --- /dev/null +++ b/tests/unit/test_data_input.py @@ -0,0 +1,55 @@ +"""Unit tests for data_input TOT_PREC de-accumulation.""" + +import numpy as np +import pytest +import xarray as xr + +from data_input import _tot_prec_handling + + +def _cumulative_tp(steps_h, values): + """Build a cumulative-from-start TOT_PREC DataArray over `steps_h` (hours).""" + step = np.array([np.timedelta64(h, "h") for h in steps_h]).astype("timedelta64[ns]") + data = np.array(values, dtype=np.float64)[:, np.newaxis] * np.ones((1, 4)) + return xr.DataArray( + data, dims=("step", "values"), coords={"step": step}, name="TOT_PREC" + ) + + +def test_tot_prec_missing_step0_synthesised_when_requested(): + """Step 0 requested but absent from the GRIB -> zero IC is synthesised.""" + tp = _cumulative_tp([6], [3.0]) + out = _tot_prec_handling(tp, requested_steps=[0, 6]) + np.testing.assert_allclose(out.sel(step=np.timedelta64(6, "h")).values, 3.0) + + +def test_tot_prec_full_range_missing_step0(): + """Full-range load with missing step 0 keeps the first lead time.""" + tp = _cumulative_tp([6, 12, 18], [3.0, 5.0, 5.5]) + out = _tot_prec_handling(tp, requested_steps=[0, 6, 12, 18]) + np.testing.assert_allclose(out.sel(step=np.timedelta64(6, "h")).values, 3.0) + np.testing.assert_allclose(out.sel(step=np.timedelta64(12, "h")).values, 2.0) + np.testing.assert_allclose(out.sel(step=np.timedelta64(18, "h")).values, 0.5) + + +def test_tot_prec_window_without_step0_untouched(): + """A [18, 24] window must not be treated as starting at an IC.""" + tp = _cumulative_tp([18, 24], [5.0, 7.0]) + out = _tot_prec_handling(tp, requested_steps=[18, 24]) + np.testing.assert_allclose(out.sel(step=np.timedelta64(24, "h")).values, 2.0) + # First window step has no preceding accumulation -> NaN after reindex. + assert np.isnan(out.sel(step=np.timedelta64(18, "h")).values).all() + + +def test_tot_prec_step0_present_but_nan_is_zero_filled(): + """Step 0 present as an all-NaN hole (earthkit allow_holes) -> zero-filled.""" + tp = _cumulative_tp([0, 6], [np.nan, 3.0]) + out = _tot_prec_handling(tp, requested_steps=[0, 6]) + np.testing.assert_allclose(out.sel(step=np.timedelta64(6, "h")).values, 3.0) + + +def test_tot_prec_single_step_without_step0_raises(): + """A single loaded step with step 0 not requested cannot be de-accumulated.""" + tp = _cumulative_tp([6], [3.0]) + with pytest.raises(ValueError, match="Cannot de-accumulate TOT_PREC"): + _tot_prec_handling(tp, requested_steps=[6]) diff --git a/workflow/scripts/verification_score_maps.py b/workflow/scripts/verification_score_maps.py index e4496ee8..445c9c52 100644 --- a/workflow/scripts/verification_score_maps.py +++ b/workflow/scripts/verification_score_maps.py @@ -365,16 +365,11 @@ def main(args: Namespace) -> None: # params request a single step. The target step is selected just # below. # - # NOTE: data_input._tot_prec_handling treats the FIRST loaded step - # positionally as the forecast initial condition and zero-fills it - # when it is present-but-all-NaN. For our window the first step is - # `step - period` (non-zero except at the first lead time); a - # corrupt/NaN field there would be silently zeroed, turning the diff - # into a cumulative-from-start value. A missing GRIB instead yields - # NaN and is skipped downstream. The principled fix belongs in - # _tot_prec_handling (gate the IC zero-fill on step 0 being - # requested) — see the port note. This applies to runs and ICON - # baselines alike. + # data_input._tot_prec_handling receives the requested steps and + # synthesises a zero initial condition when step 0 is requested but + # absent from the GRIB (anemoi-inference omits TOT_PREC at step 0), + # which makes the first-lead-time window [0, period] work for ML + # runs. Windows not containing step 0 are never zero-filled. src_root = args.baseline_root if args.baseline_root else grib_dir fcst = load_forecast_data( src_root, reftime, req_steps, fct_params, member=args.member From 3fe25315d43de9367f8c2ac2020f6bd076a05fcb Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 15 Jun 2026 10:14:48 +0200 Subject: [PATCH 124/129] Fix run-side score-maps rule: ECCODES definitions and output_root The verification_score_maps rule was missing the COSMO eccodes definitions export that every other GRIB-reading rule sets, so forecast variable selection found nothing and run-side score maps came out empty. It also hardcoded --run_root to output/, ignoring locations.output_root; derive it from OUT_ROOT like verification_metrics does. Co-Authored-By: Claude Opus 4.8 (1M context) --- workflow/rules/verification.smk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index 52032864..41ea094e 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -198,10 +198,12 @@ rule verification_score_maps: fcst_steps=lambda wc: RUN_CONFIGS[wc.run_id]["steps"], truth_label=config["truth"]["label"], reftimes=" ".join(t.strftime("%Y%m%d%H%M") for t in REFTIMES), + run_root=lambda wc: (Path(OUT_ROOT) / f"data/runs/{wc.run_id}").resolve(), shell: """ + export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) uv run {input.script} \ - --run_root output/data/runs/{wildcards.run_id} \ + --run_root {params.run_root} \ --reftimes {params.reftimes} \ --truth {input.truth} \ --step {wildcards.leadtime} \ From 780cae8054b26a15b6996d115965c89642407388 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 15 Jun 2026 10:14:49 +0200 Subject: [PATCH 125/129] Fix score-map plotting after earthkit/loader migration Two post-merge breakages: the score-map netCDFs now carry latitude/longitude (not lat/lon) since the loader port, and earthkit-plots >= 1.0 rejects Matplotlib Colormap objects for Style.colors. Read the new coordinate names and sample colormaps into explicit colour lists. Co-Authored-By: Claude Opus 4.8 (1M context) --- workflow/scripts/plot_score_maps.mo.py | 28 +++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/workflow/scripts/plot_score_maps.mo.py b/workflow/scripts/plot_score_maps.mo.py index c484ad3e..be07a3fb 100644 --- a/workflow/scripts/plot_score_maps.mo.py +++ b/workflow/scripts/plot_score_maps.mo.py @@ -120,9 +120,13 @@ def _(LOG, init_hour, param, score, season, verif_file, xr): def _(CMAP_DEFAULTS, ekp): def get_style(param, score, units_override=None): """Get style and colormap settings for the plot. - Needed because cmap/norm does not work in Style(colors=cmap), - still needs to be passed as arguments to tripcolor()/tricontourf(). + + earthkit-plots >= 1.0 expects ``Style.colors`` to be a list of + colours; Matplotlib ``Colormap`` objects from CMAP_DEFAULTS are + sampled into one colour per level interval. """ + from matplotlib import colors as mcolors + score_key = f"{param}.{score}.map" cfg = ( CMAP_DEFAULTS[score_key] @@ -130,19 +134,19 @@ def get_style(param, score, units_override=None): else CMAP_DEFAULTS.get(param, {}) ) units = units_override if units_override is not None else cfg.get("units", "") + levels = cfg.get("bounds", cfg.get("levels", None)) + colors = cfg.get("colors", None) + cmap = cfg.get("cmap", None) + if colors is None and cmap is not None: + n = len(levels) - 1 if levels is not None else getattr(cmap, "N", 256) + colors = [mcolors.to_hex(cmap(i / max(n - 1, 1))) for i in range(n)] return { "style": ekp.styles.Style( - levels=cfg.get("bounds", cfg.get("levels", None)), + levels=levels, extend="both", units=units, - colors=cfg.get("colors", None), + colors=colors, ), - "norm": cfg.get("norm", None), - "cmap": cfg.get("cmap", None), - "levels": cfg.get("levels", None), - "vmin": cfg.get("vmin", None), - "vmax": cfg.get("vmax", None), - "colors": cfg.get("colors", None), } return (get_style,) @@ -167,8 +171,8 @@ def _( # plot individual fields plotter = StatePlotter( - ds["lon"].values.ravel(), - ds["lat"].values.ravel(), + ds["longitude"].values.ravel(), + ds["latitude"].values.ravel(), outfn.parent, ) fig = plotter.init_geoaxes( From 6e7d5915e93a348b15932faa7d06aff527bdece3 Mon Sep 17 00:00:00 2001 From: Louis-Frey Date: Mon, 15 Jun 2026 11:39:05 +0200 Subject: [PATCH 126/129] Resolve score-map lead times per participant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolve_leadtimes returned a participant-agnostic union of every configured run's and baseline's steps, so 'leadtimes: all' (and explicit lists) scheduled jobs participants could not satisfy — e.g. a 36h map for an ICON-CH1 baseline (steps 0/33/6), or a 0h precip map (no accumulation window). Resolve lead times against each participant's own steps spec instead: keep only the requested lead times it produces, warn and skip those it does not, and drop sub-period lead times for accumulated params. Build the score_maps_all targets per (participant, param), and reuse the same function for the showcase get_leadtimes, dropping its duplicated parse. --- workflow/Snakefile | 58 ++++++++++++++++++++++++--------------- workflow/rules/common.smk | 47 +++++++++++++++++++++---------- workflow/rules/plot.smk | 9 ++---- 3 files changed, 72 insertions(+), 42 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 24214458..98b40653 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -133,31 +133,45 @@ onerror: SCORE_MAPS_CONFIG = config["experiment"]["score_maps"] +def _score_map_targets(): + """Score-map plot targets, expanded per participant. + + Lead times are resolved per (participant, param) via resolve_leadtimes, so + each run/baseline only contributes lead times its own forecast steps + produce — 'leadtimes: all' and explicit lists never schedule a job a + participant cannot satisfy. + """ + requested = SCORE_MAPS_CONFIG["leadtimes"] + common = dict( + score=SCORE_MAPS_CONFIG["scores"], + region=SCORE_MAPS_CONFIG["regions"], + season=SCORE_MAPS_CONFIG["seasons"], + init_hour=SCORE_MAPS_CONFIG["init_hours"], + experiment=EXPERIMENT_NAME, + ) + targets = [] + for out_rule, participants in ( + (rules.plot_score_maps.output, ("run_id", collect_all_candidates())), + (rules.plot_score_maps_baseline.output, ("baseline_id", BASELINE_CONFIGS)), + ): + wildcard, configs = participants + for participant_id, cfg in configs.items(): + for param in SCORE_MAPS_CONFIG["params"]: + leadtimes = resolve_leadtimes(cfg["steps"], requested, param) + targets += expand( + out_rule, + param=param, + leadtime=leadtimes, + **{wildcard: participant_id}, + **common, + ) + return targets + + rule score_maps_all: """Target rule for score maps (opt-in via experiment.score_maps.enabled).""" input: - expand( - rules.plot_score_maps.output, - run_id=collect_all_candidates(), - leadtime=resolve_leadtimes(SCORE_MAPS_CONFIG["leadtimes"]), - score=SCORE_MAPS_CONFIG["scores"], - param=SCORE_MAPS_CONFIG["params"], - region=SCORE_MAPS_CONFIG["regions"], - season=SCORE_MAPS_CONFIG["seasons"], - init_hour=SCORE_MAPS_CONFIG["init_hours"], - experiment=EXPERIMENT_NAME, - ), - expand( - rules.plot_score_maps_baseline.output, - baseline_id=list(BASELINE_CONFIGS), - leadtime=resolve_leadtimes(SCORE_MAPS_CONFIG["leadtimes"]), - score=SCORE_MAPS_CONFIG["scores"], - param=SCORE_MAPS_CONFIG["params"], - region=SCORE_MAPS_CONFIG["regions"], - season=SCORE_MAPS_CONFIG["seasons"], - init_hour=SCORE_MAPS_CONFIG["init_hours"], - experiment=EXPERIMENT_NAME, - ), + _score_map_targets(), rule experiment_all: diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index a62ff46b..190a09c5 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -355,18 +355,37 @@ SCORECARD_CONFIGS = ( ) -def resolve_leadtimes(spec): - """Resolve a lead-time specification from config. - - Accepts: - - a list of ints — returned verbatim. - - the literal string "all" — expanded to the union of step lists - from all configured runs and baselines. +# Period-accumulated params verify a [lead - period, lead] window, so they have +# no value at lead times shorter than one step spacing (e.g. no 0h precip map). +# Short and canonical names both appear across the workflow (showcases vs maps). +ACCUMULATED_PARAMS = {"TOT_PREC", "tp"} + + +def resolve_leadtimes(steps_spec, requested="all", param=None): + """Lead times to compute for a single participant. + + A run or baseline produces only the lead times in its own ``steps`` spec + (``start/stop/step``, hours). This returns those of the ``requested`` + selection that the participant actually produces — the literal ``"all"`` + (every produced lead time) or an explicit list of ints — so a 36h lead is + never requested of an ICON-CH1 baseline (steps ``0/33/6``), nor a >120h + lead of ICON-CH2. Explicitly requested lead times the participant cannot + produce are skipped with a warning. For accumulated ``param``s, lead times + shorter than one step spacing are dropped (no accumulation window). """ - if spec != "all": - return spec - all_steps = set() - for cfg in (*RUN_CONFIGS.values(), *BASELINE_CONFIGS.values()): - start, end, step = map(int, cfg["steps"].split("/")) - all_steps.update(range(start, end + 1, step)) - return sorted(all_steps) + start, end, step = map(int, steps_spec.split("/")) + supported = set(range(start, end + 1, step)) + wanted = supported if requested == "all" else set(requested) + + unsupported = sorted(wanted - supported) + if unsupported: + logging.getLogger("snakemake").warning( + "Skipping lead time(s) %sh: not produced by forecast steps '%s'.", + unsupported, + steps_spec, + ) + + valid = wanted & supported + if param in ACCUMULATED_PARAMS: + valid = {lt for lt in valid if lt >= step} + return sorted(valid) diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 1392a53d..13ef3d25 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -132,12 +132,9 @@ rule plot_forecast_frame: def get_leadtimes(wc): - """Get all lead times from the run config.""" - start, end, step = map(int, RUN_CONFIGS[wc.run_id]["steps"].split("/")) - # skip lead time 0 for diagnostic variables - if wc.param in ["tp", "TOT_PREC"] and start == 0: - start += step - return [f"{i}" for i in range(start, end + 1, step)] + """Get all lead times the run produces (accumulated params skip lead 0).""" + leadtimes = resolve_leadtimes(RUN_CONFIGS[wc.run_id]["steps"], param=wc.param) + return [str(lt) for lt in leadtimes] rule make_forecast_animation: From 509808e7821994be19292d1f2125c5c4bd67de4a Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Mon, 15 Jun 2026 16:43:27 +0200 Subject: [PATCH 127/129] Simplify expansion of score map outputs --- workflow/Snakefile | 72 +++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 42 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 98b40653..1438c56f 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -133,47 +133,6 @@ onerror: SCORE_MAPS_CONFIG = config["experiment"]["score_maps"] -def _score_map_targets(): - """Score-map plot targets, expanded per participant. - - Lead times are resolved per (participant, param) via resolve_leadtimes, so - each run/baseline only contributes lead times its own forecast steps - produce — 'leadtimes: all' and explicit lists never schedule a job a - participant cannot satisfy. - """ - requested = SCORE_MAPS_CONFIG["leadtimes"] - common = dict( - score=SCORE_MAPS_CONFIG["scores"], - region=SCORE_MAPS_CONFIG["regions"], - season=SCORE_MAPS_CONFIG["seasons"], - init_hour=SCORE_MAPS_CONFIG["init_hours"], - experiment=EXPERIMENT_NAME, - ) - targets = [] - for out_rule, participants in ( - (rules.plot_score_maps.output, ("run_id", collect_all_candidates())), - (rules.plot_score_maps_baseline.output, ("baseline_id", BASELINE_CONFIGS)), - ): - wildcard, configs = participants - for participant_id, cfg in configs.items(): - for param in SCORE_MAPS_CONFIG["params"]: - leadtimes = resolve_leadtimes(cfg["steps"], requested, param) - targets += expand( - out_rule, - param=param, - leadtime=leadtimes, - **{wildcard: participant_id}, - **common, - ) - return targets - - -rule score_maps_all: - """Target rule for score maps (opt-in via experiment.score_maps.enabled).""" - input: - _score_map_targets(), - - rule experiment_all: """Target rule for experiment workflow.""" input: @@ -202,7 +161,36 @@ rule experiment_all: ) else [] ), - (rules.score_maps_all.input if SCORE_MAPS_CONFIG["enabled"] else []), + ( + expand( + rules.plot_score_maps.output, + run_id=list(CANDIDATES), + param=SCORE_MAPS_CONFIG["params"], + leadtime=SCORE_MAPS_CONFIG["leadtimes"], + score=SCORE_MAPS_CONFIG["scores"], + region=SCORE_MAPS_CONFIG["regions"], + season=SCORE_MAPS_CONFIG["seasons"], + init_hour=SCORE_MAPS_CONFIG["init_hours"], + experiment=EXPERIMENT_NAME, + ) + if SCORE_MAPS_CONFIG["enabled"] + else [] + ), + ( + expand( + rules.plot_score_maps_baseline.output, + baseline_id=list(BASELINES), + param=SCORE_MAPS_CONFIG["params"], + leadtime=SCORE_MAPS_CONFIG["leadtimes"], + score=SCORE_MAPS_CONFIG["scores"], + region=SCORE_MAPS_CONFIG["regions"], + season=SCORE_MAPS_CONFIG["seasons"], + init_hour=SCORE_MAPS_CONFIG["init_hours"], + experiment=EXPERIMENT_NAME, + ) + if SCORE_MAPS_CONFIG["enabled"] + else [] + ), rule showcase_all: From f38c2ee0b8592ecbe19374476d5dcfea9cdc21cb Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Mon, 15 Jun 2026 16:52:13 +0200 Subject: [PATCH 128/129] Remove support for 'leadtimes=all' --- src/evalml/config.py | 10 +++------- workflow/tools/config.schema.json | 20 ++++++-------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/src/evalml/config.py b/src/evalml/config.py index b5b42402..2021634b 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, List, Any, ClassVar, FrozenSet, Literal, Optional +from typing import Dict, List, Any, ClassVar, FrozenSet, Optional from pydantic import BaseModel, Field, RootModel, field_validator @@ -233,13 +233,9 @@ class ScoreMapsConfig(BaseModel): "PS, PMSL, TOT_PREC (native), and SP_10M (derived wind speed from U_10M/V_10M)." ), ) - leadtimes: List[int] | Literal["all"] = Field( + leadtimes: List[int] = Field( default=[6, 24], - description=( - "List of lead times (hours) to plot, or the literal string 'all' " - "to expand to the union of step lists from all configured runs " - "and baselines." - ), + description="List of lead times (hours) to plot.", ) scores: List[str] = Field( default=["BIAS"], diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index 8e677694..b4ea0d49 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -591,24 +591,16 @@ "type": "array" }, "leadtimes": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "const": "all", - "type": "string" - } - ], "default": [ 6, 24 ], - "description": "List of lead times (hours) to plot, or the literal string 'all' to expand to the union of step lists from all configured runs and baselines.", - "title": "Leadtimes" + "description": "List of lead times (hours) to plot.", + "items": { + "type": "integer" + }, + "title": "Leadtimes", + "type": "array" }, "scores": { "default": [ From 202b651ef3917a1a7750be96e0d62c41507dbc10 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Mon, 15 Jun 2026 16:56:49 +0200 Subject: [PATCH 129/129] Add validator for leadtimes in score maps --- src/evalml/config.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/evalml/config.py b/src/evalml/config.py index 2021634b..b2b0cb20 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Dict, List, Any, ClassVar, FrozenSet, Optional -from pydantic import BaseModel, Field, RootModel, field_validator +from pydantic import BaseModel, Field, RootModel, field_validator, model_validator PROJECT_ROOT = Path(__file__).parents[2] @@ -535,6 +535,24 @@ class ConfigModel(BaseModel): description="Settings for the showcase workflow.", ) + @model_validator(mode="after") + def validate_score_map_leadtimes(self) -> "ConfigModel": + sm = self.experiment.score_maps + if not sm.enabled: + return self + requested = set(sm.leadtimes) + for item in self.runs: + steps = getattr(item, next(iter(item.model_fields))).steps + start, end, step = map(int, steps.split("/")) + producible = set(range(start, end + 1, step)) + unsupported = requested - producible + if unsupported: + raise ValueError( + f"score_maps.leadtimes contains {sorted(unsupported)} h which are not " + f"produced by participant with steps '{steps}'." + ) + return self + model_config = { "extra": "forbid", # fail on misspelled keys "populate_by_name": True,