feat: train on replayed rollouts — continue (compaction/tool-call) + recheck envs

configs/debug/v1/gsm8k_continue_tool.toml

@@ -0,0 +1,55 @@
+# GSM8K (v1) + online continue replay — gsm8k.toml plus a second train env that re-enters this
+# run's own finished attempts mid-conversation, anchored at a tool call (the default harness
+# gives the model bash+edit tools, so source rollouts contain real tool calls; gsm8k is
+# stateless so replay fidelity is exact). The replay env starts empty and grows as steps ship
+# records into <output_dir>/rollouts/. Rollouts with no tool calls are skipped (no_seed), so
+# the replay taskset may be a small fraction of the base env's records.
+#
+# Derived from configs/debug/v1/gsm8k_recheck.toml. Runs on 2 GPUs (1 inference, 1 training):
+#   uv run inference --model.name PrimeIntellect/Qwen3-0.6B          # Inference pane (GPU 0)
+#   uv run rl @ configs/debug/v1/gsm8k_continue_tool.toml --ckpt     # Trainer pane (GPU 1)
+
+max_steps = 100
+seq_len = 2048
+output_dir = "outputs-continue"
+
+[wandb]
+project = "gsm8k-debug"
+name = "gsm8k-continue-v1"
+
+[model]
+name = "Qwen/Qwen3-0.6B"
+
+[orchestrator]
+batch_size = 512
+group_size = 16
+
+[orchestrator.train.sampling]
+max_completion_tokens = 2048
+
+[[orchestrator.train.env]]
+taskset = { id = "gsm8k-v1", split = "train" }
+harness = { id = "default", runtime = { type = "subprocess" } }
+
+# `records` must point at THIS run's own records (note the run-id subdir the
+# orchestrator nests under output_dir) — adjust if you launch with --output-dir.
+[[orchestrator.train.env]]
+name = "gsm8k-continue"
+taskset = { id = "replay", records = "outputs-continue/run_default/rollouts/step_*/train_rollouts_gsm8k-v1.jsonl", mode = "continue", anchor = "tool-call", source = { id = "gsm8k-v1", split = "train" }, max_seed_tokens = 4096 }
+harness = { id = "default", runtime = { type = "subprocess" } }
+ratio = 1
+
+[orchestrator.eval]
+interval = 20
+num_examples = 256
+
+[orchestrator.eval.sampling]
+max_completion_tokens = 2048
+
+[[orchestrator.eval.env]]
+taskset = { id = "gsm8k-v1", split = "test" }
+harness = { id = "null", runtime = { type = "subprocess" } }
+
+[trainer] # Default trainer config
+
+[inference] # Default inference config

configs/debug/v1/gsm8k_recheck.toml

@@ -0,0 +1,51 @@
+# GSM8K (v1) + online recheck replay — configs/debug/v1/gsm8k.toml plus a second train env
+# that re-enters this run's own finished attempts with a verification turn (scored by the
+# gsm8k verifier; no sandbox state involved, so replay fidelity is exact). The replay env
+# starts empty and grows as steps ship records into <output_dir>/rollouts/. Runs on 2 GPUs
+# (1 inference, 1 training):
+#   uv run inference --model.name PrimeIntellect/Qwen3-0.6B          # Inference pane (GPU 0)
+#   uv run rl @ configs/debug/v1/gsm8k_recheck.toml --ckpt           # Trainer pane (GPU 1)
+
+max_steps = 100
+seq_len = 2048
+
+[wandb]
+project = "gsm8k-debug"
+name = "gsm8k-v1"
+
+[model]
+name = "Qwen/Qwen3-0.6B"
+
+[orchestrator]
+batch_size = 512
+group_size = 16
+
+[orchestrator.train.sampling]
+max_completion_tokens = 2048
+
+[[orchestrator.train.env]]
+taskset = { id = "gsm8k-v1", split = "train" }
+harness = { id = "null", runtime = { type = "subprocess" } }
+
+# `records` must point at THIS run's own records (note the run-id subdir the
+# orchestrator nests under output_dir) — adjust if you launch with --output-dir.
+[[orchestrator.train.env]]
+name = "gsm8k-recheck"
+taskset = { id = "replay", records = "outputs/run_default/rollouts/step_*/train_rollouts_gsm8k-v1.jsonl", mode = "recheck", source = { id = "gsm8k-v1", split = "train" }, max_seed_tokens = 4096 }
+harness = { id = "null", runtime = { type = "subprocess" } }
+ratio = 1
+
+[orchestrator.eval]
+interval = 20
+num_examples = 256
+
+[orchestrator.eval.sampling]
+max_completion_tokens = 2048
+
+[[orchestrator.eval.env]]
+taskset = { id = "gsm8k-v1", split = "test" }
+harness = { id = "null", runtime = { type = "subprocess" } }
+
+[trainer] # Default trainer config
+
+[inference] # Default inference config

configs/debug/v1/gsm8k_recheck_incorrect.toml

@@ -0,0 +1,52 @@
+output_dir = "outputs-wrong"
+# GSM8K (v1) + online recheck replay — configs/debug/v1/gsm8k.toml plus a second train env
+# that re-enters this run's own finished attempts with a verification turn (scored by the
+# gsm8k verifier; no sandbox state involved, so replay fidelity is exact). The replay env
+# starts empty and grows as steps ship records into <output_dir>/rollouts/. Runs on 2 GPUs
+# (1 inference, 1 training):
+#   uv run inference --model.name PrimeIntellect/Qwen3-0.6B          # Inference pane (GPU 0)
+#   uv run rl @ configs/debug/v1/gsm8k_recheck_incorrect.toml --ckpt # Trainer pane (GPU 1)
+
+max_steps = 100
+seq_len = 2048
+
+[wandb]
+project = "gsm8k-debug"
+name = "gsm8k-v1"
+
+[model]
+name = "Qwen/Qwen3-0.6B"
+
+[orchestrator]
+batch_size = 512
+group_size = 16
+
+[orchestrator.train.sampling]
+max_completion_tokens = 2048
+
+[[orchestrator.train.env]]
+taskset = { id = "gsm8k-v1", split = "train" }
+harness = { id = "null", runtime = { type = "subprocess" } }
+
+# `records` must point at THIS run's own records (note the run-id subdir the
+# orchestrator nests under output_dir) — adjust if you launch with --output-dir.
+[[orchestrator.train.env]]
+name = "gsm8k-recheck-wrong"
+taskset = { id = "replay", records = "outputs-wrong/run_default/rollouts/step_*/train_rollouts_gsm8k-v1.jsonl", mode = "recheck", max_source_reward = 0.5, source = { id = "gsm8k-v1", split = "train" }, max_seed_tokens = 4096 }
+harness = { id = "null", runtime = { type = "subprocess" } }
+ratio = 1
+
+[orchestrator.eval]
+interval = 20
+num_examples = 256
+
+[orchestrator.eval.sampling]
+max_completion_tokens = 2048
+
+[[orchestrator.eval.env]]
+taskset = { id = "gsm8k-v1", split = "test" }
+harness = { id = "null", runtime = { type = "subprocess" } }
+
+[trainer] # Default trainer config
+
+[inference] # Default inference config

docs/configuration.md

@@ -18,6 +18,7 @@ Every `prime-rl` entrypoint uses [`pydantic-config`](https://github.com/PrimeInt
   - [None](#none)
   - [Discriminated Unions](#discriminated-unions)
   - [Environments](#environments-orchestratortrainenv)
+  - [Replaying Past Rollouts (Continue / Recheck)](#replaying-past-rollouts-continue--recheck)
   - [Environment Variables](#environment-variables)
 - [Examples](#examples)
 
@@ -169,6 +170,38 @@ args = { dataset_name = "openai/gsm8k", dataset_subset = "main" }
 
 The same `id` can appear multiple times across train and eval (or with different `args`) — useful for evaluating on a held-out split of the env you're training on, or comparing two configurations of the same env side by side. When `id` is reused, set a distinct `name` on each entry; `name` defaults to `id` and must be unique across all envs in the same group.
 
+### Replaying Past Rollouts (Continue / Recheck)
+
+The built-in `replay` taskset (verifiers v1) re-enters saved rollouts as fresh training tasks. Point it at rollout record files (`<output_dir>/rollouts/step_*/*_rollouts_*.jsonl`) and at the `source` taskset the records came from — the source provides tools, setup/finalize, and scoring, so new completions are judged by the original env's verifier:
+
+```toml
+# Continue: resume each recorded compaction from its handoff summary. The seed is a plain
+# string prompt, so it runs under any harness — use the same harness config as the source run.
+[[orchestrator.train.env]]
+name = "my-env-continue"
+taskset = { id = "replay", records = "/data/run1/rollouts/step_*/train_rollouts_<env>.jsonl", mode = "continue", source = { id = "my-env-v1" } }
+harness = { id = "rlm" }
+
+# Recheck: replay each finished attempt with a "check your work" turn appended. The seed is a
+# full conversation, which needs a message-seeding harness (`default`/`null`).
+[[orchestrator.train.env]]
+name = "my-env-recheck"
+taskset = { id = "replay", records = "/data/run1/rollouts/step_*/train_rollouts_<env>.jsonl", mode = "recheck", source = { id = "my-env-v1" } }
+harness = { id = "null" }
+```
+
+Key knobs on the `taskset` table:
+
+- `mode = "continue"` resumes a rollout mid-way; `anchor` picks the resume point: `"compaction"` (default; one task per recorded context restart, detected structurally so records from different harnesses mix freely) or `"tool-call"` (deterministically-drawn resume points right after complete tool-result runs; needs a message-seeding harness). For `"tool-call"`, `max_anchors` sets how many resume points each source rollout seeds (default 1; `None` seeds every valid one, in trajectory order).
+- `mode = "recheck"` replays the final branch of each attempt (truncation artifacts stripped) plus a verification turn (`recheck_prompt` overrides the wording).
+- `max_seed_tokens` skips seeds whose context exceeds the budget — set it so seeds leave room to sample under the trainer's `seq_len`.
+- `records` globs are followed: new matching files are picked up continuously (append-only, so pool workers stay index-aligned). Point it at a finished run's records, or at the *current* run's own `rollouts/` dir for online self-replay — the env starts empty and grows as steps ship; the replay env's own rollouts are never re-mined as seeds. Avoid sources that grow out of glob order (e.g. several runs writing one tree).
+- Lines that don't validate as the source taskset's task type (e.g. other envs' rollouts in a mixed `train_rollouts_<env>.jsonl`) are skipped and counted. The filter is structural: for a source whose task type has no distinguishing required fields, use single-env record files instead.
+
+The orchestrator writes a derived `index_<env>.jsonl` beside each step's `train_rollouts_<env>.jsonl`: one row per record with selection fields (task name, source reward, branch count, ...) and the record's byte span. When an index is present, replay filters records index-side — already-replayed tasks, out-of-range source rewards, and (for `continue`/`compaction`) un-compacted rollouts — and parses only the selected spans; a record file without an index (e.g. hand-assembled) is fully parsed instead. The index is optional and derived; the record files themselves are unchanged.
+
+Group rollouts of one seeded task form a regular GRPO group, so a group-relative algorithm gets contrastive signal at exactly the resumed state. The sandbox is fresh on re-entry: `setup` runs anew, and no filesystem state from the source rollout is replayed. Records carrying sandbox snapshot refs (`trace.info["snapshots"]`) restrict resume points to snapshotted anchors and restore the ref during `setup` — no runtime implements snapshot capture yet, so restore fails loudly if refs ever appear before support lands. Other recycling schemes can subclass `ReplayTaskset` and override `seeds()` — see `deps/verifiers/verifiers/v1/tasksets/replay/`.
+
 ### Environment Variables
 
 OS environment variables exported into launched component process(es). In `rl` configs, top-level `[env_vars]` applies to trainer, inference, and orchestrator:

outputs-continue/configs/inference.toml

@@ -0,0 +1,44 @@
+enable_lora = false
+max_loras = 8
+max_cpu_loras = 100
+gpu_memory_utilization = 0.9
+api_server_count = 1
+data_parallel_rpc_port = 13345
+seed = 0
+enable_expert_parallel = false
+all2all_backend = "allgather_reducescatter"
+enable_eplb = false
+enable_dbo = false
+use_deep_gemm = false
+use_pd_kv_transfer = false
+enable_return_routed_experts = false
+enable_fp32_lm_head = true
+
+[server]
+port = 8000
+liveness_timeout_seconds = 30.0
+
+[model]
+name = "Qwen/Qwen3-0.6B"
+trust_remote_code = false
+dtype = "auto"
+enforce_eager = false
+tool_call_parser = "hermes"
+
+[parallel]
+tp = 1
+dp = 1
+
+[log]
+level = "info"
+vf_level = "info"
+json_logging = false
+log_data = false
+interval = 10.0
+
+[env_vars]
+
+[weight_broadcast]
+type = "filesystem"
+
+[vllm_extra]