From d7020e98d3ac8fa5b7e3a82dfa2ae2888c2ee6bf Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:48:52 +0200 Subject: [PATCH 1/3] Add reusable OpenEnv taskset support --- environments/README.md | 3 +- environments/openenv_echo_v1/README.md | 28 +++ .../openenv_echo_v1/__init__.py | 3 + .../openenv_echo_v1/taskset.py | 22 ++ environments/openenv_echo_v1/pyproject.toml | 13 ++ tests/v1/test_envs.py | 7 +- tests/v1/test_openenv.py | 205 ++++++++++++++++++ verifiers/v1/GUIDE.md | 4 + verifiers/v1/README.md | 28 ++- verifiers/v1/__init__.py | 4 + verifiers/v1/harnesses/codex/harness.py | 20 +- verifiers/v1/mcp/__init__.py | 9 +- verifiers/v1/mcp/toolset.py | 107 ++++++++- verifiers/v1/runtimes/base.py | 3 +- verifiers/v1/tasksets/__init__.py | 8 +- verifiers/v1/tasksets/openenv/__init__.py | 3 + verifiers/v1/tasksets/openenv/taskset.py | 62 ++++++ 17 files changed, 511 insertions(+), 18 deletions(-) create mode 100644 environments/openenv_echo_v1/README.md create mode 100644 environments/openenv_echo_v1/openenv_echo_v1/__init__.py create mode 100644 environments/openenv_echo_v1/openenv_echo_v1/taskset.py create mode 100644 environments/openenv_echo_v1/pyproject.toml create mode 100644 tests/v1/test_openenv.py create mode 100644 verifiers/v1/tasksets/openenv/__init__.py create mode 100644 verifiers/v1/tasksets/openenv/taskset.py diff --git a/environments/README.md b/environments/README.md index da47ab0b0d..e56f305a48 100644 --- a/environments/README.md +++ b/environments/README.md @@ -24,6 +24,7 @@ This folder contains installable example environments that showcase common usage - **wordle**: Game-style interaction via `TextArenaEnv`; multiple rewards (correctness, partial credit, few-turn bonus) and XML formatting. - **wordle_v1**: Wordle on the reusable v1 `TextArenaTaskset`, with Wordle-specific prompt, feedback, and rewards kept in the environment package. - **openenv_echo**: OpenEnv MCP integration example using upstream `echo_env`. +- **openenv_echo_v1**: Thin v1 config over the reusable OpenEnv image taskset, pinned to the official Echo image. - **openenv_textarena**: OpenEnv gym integration example using upstream `textarena_env` (default `Wordle-v0`). ### Tool use @@ -91,7 +92,7 @@ This folder contains installable example environments that showcase common usage - **ToolEnv with real tools**: `wiki_search`, `math_python` - **Custom MultiTurnEnv**: `alphabet_sort`, `doublecheck`, `sentence_repeater`, `wordle` - **GymEnv integration**: `gem_wordle` -- **OpenEnv integration (gym + MCP)**: `openenv_textarena`, `openenv_echo` +- **OpenEnv integration (gym + MCP)**: `openenv_textarena`, `openenv_echo`, `openenv_echo_v1` - **CLI agent sandboxes**: `opencode_harbor`, `terminus_harbor`, `hello_mcp_harbor` - **MCP integration**: `mcp_search_env`, `hello_mcp_harbor` - **Taskset/Harness v1**: use this pattern for new environments that need reusable tasksets, reusable harnesses, framework programs, endpoint interception, or sandboxed Python/command programs. Examples include `dspy_rlm`, `openai_agents_env`, `langchain_deep_agents_wikispeedia`, `reverse_text`, `alphabet_sort`, `wiki_search`, `math_python`, `mcp_search_env`, `opencode_harbor`, `bfcl_v3`, `hello_subagent_v1`, `nested_harness_v1`, `hello_self_judge_v1`, `hello_parallel_sandbox_v1`, `hello_group_reward_v1`, `hello_rlm_v1`, `rlm_swe_v1`, `dspy_flights`, and `wordle-v1`. diff --git a/environments/openenv_echo_v1/README.md b/environments/openenv_echo_v1/README.md new file mode 100644 index 0000000000..4d67897ed8 --- /dev/null +++ b/environments/openenv_echo_v1/README.md @@ -0,0 +1,28 @@ +# openenv-echo-v1 + +OpenEnv's official Echo image as a minimal v1 example. The reusable image lifecycle and +JSON-RPC-to-MCP bridge live in `verifiers`; this package only pins the image, prompt, and +resources. + +## Develop + +1. From the Verifiers checkout root, add both the current framework and this example to the + ephemeral `uv run` environment. +2. Run it with any MCP-capable Verifiers harness and a container runtime: + +```bash +uv run --with-editable . --with-editable environments/openenv_echo_v1 \ + eval openenv-echo-v1 -n 1 --harness.runtime.type docker +uv run --with-editable . --with-editable environments/openenv_echo_v1 \ + eval openenv-echo-v1 -n 1 --harness.runtime.type docker --harness.id rlm +uv run --with-editable . --with-editable environments/openenv_echo_v1 \ + eval openenv-echo-v1 -n 1 --harness.runtime.type docker --harness.id codex \ + -m nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B +``` + +## Layout + +- `openenv_echo_v1/taskset.py` — a thin config over `vf`'s reusable `OpenEnvTaskset`. + +Echo's production MCP contract is unscored, so the reward is neutral while the tool call and +result remain in the Verifiers trace. diff --git a/environments/openenv_echo_v1/openenv_echo_v1/__init__.py b/environments/openenv_echo_v1/openenv_echo_v1/__init__.py new file mode 100644 index 0000000000..7cffea7c23 --- /dev/null +++ b/environments/openenv_echo_v1/openenv_echo_v1/__init__.py @@ -0,0 +1,3 @@ +from openenv_echo_v1.taskset import OpenEnvEchoTaskset + +__all__ = ["OpenEnvEchoTaskset"] diff --git a/environments/openenv_echo_v1/openenv_echo_v1/taskset.py b/environments/openenv_echo_v1/openenv_echo_v1/taskset.py new file mode 100644 index 0000000000..b01f466201 --- /dev/null +++ b/environments/openenv_echo_v1/openenv_echo_v1/taskset.py @@ -0,0 +1,22 @@ +"""OpenEnv's official Echo image as a zero-config example taskset.""" + +import verifiers.v1 as vf +from verifiers.v1.tasksets.openenv import OpenEnvConfig, OpenEnvTaskset + +ECHO_IMAGE = ( + "ghcr.io/meta-pytorch/openenv-echo-env@" + "sha256:56c55669c00b23a6af6adbcd8dd1fb5da3a276aec186b5c46cb4abeb708afa9c" +) + + +class OpenEnvEchoConfig(OpenEnvConfig): + image: str = ECHO_IMAGE + prompt: str = ( + 'Call the echo_message tool with the message "Hello, World!", then return ' + "the echoed text." + ) + resources: vf.TaskResources = vf.TaskResources(cpu=2, memory=4, disk=10) + + +class OpenEnvEchoTaskset(OpenEnvTaskset, vf.Taskset[vf.Task, OpenEnvEchoConfig]): + pass diff --git a/environments/openenv_echo_v1/pyproject.toml b/environments/openenv_echo_v1/pyproject.toml new file mode 100644 index 0000000000..87e3212050 --- /dev/null +++ b/environments/openenv_echo_v1/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "openenv-echo-v1" +version = "0.1.0" +description = "openenv-echo-v1 — ." +requires-python = ">=3.11" +dependencies = ["verifiers"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["openenv_echo_v1"] diff --git a/tests/v1/test_envs.py b/tests/v1/test_envs.py index 61ace875ba..a01aa895c2 100644 --- a/tests/v1/test_envs.py +++ b/tests/v1/test_envs.py @@ -20,9 +20,10 @@ ENVIRONMENTS = Path(__file__).parent.parent.parent / "environments" # v1 tasksets that can't run a plain-CI smoke eval — e.g. they need a docker/prime runtime or -# clone a corpus CI can't read. Empty: the SWE/container and corpus tasksets live in -# research-environments now. -SKIP_EVAL: set[str] = set() +# clone a corpus CI can't read. +SKIP_EVAL = { + "openenv_echo_v1", # requires its pinned OpenEnv container image +} def v1_tasksets() -> list[str]: diff --git a/tests/v1/test_openenv.py b/tests/v1/test_openenv.py new file mode 100644 index 0000000000..3bb1d95160 --- /dev/null +++ b/tests/v1/test_openenv.py @@ -0,0 +1,205 @@ +from pathlib import Path +from unittest.mock import AsyncMock, Mock + +import httpx +import pytest +from mcp.server.fastmcp import FastMCP + +import verifiers.v1 as vf +from verifiers.v1.env import EnvConfig, Environment +from verifiers.v1.runtimes import ProgramResult +from verifiers.v1.runtimes.subprocess import SubprocessConfig, SubprocessRuntime +from verifiers.v1.tasksets.openenv import OpenEnvConfig, OpenEnvTaskset +from verifiers.v1.tasksets.openenv.taskset import ( + OPENENV_SOCKET, + OPENENV_URL, +) + +ECHO_EXAMPLE = Path(__file__).parents[2] / "environments" / "openenv_echo_v1" + + +@pytest.mark.asyncio +async def test_uv_script_ignores_image_system_python() -> None: + runtime = SubprocessRuntime(SubprocessConfig()) + runtime.write = AsyncMock() + runtime.run = AsyncMock( + side_effect=[ + ProgramResult(exit_code=0, stdout="", stderr=""), + ProgramResult( + exit_code=0, stdout="/tmp/openenv-test/bin/python\n", stderr="" + ), + ] + ) + + await runtime.prepare_uv_script("# openenv image system-python regression\n") + + prepare_command = runtime.run.await_args_list[1].args[0][2] + assert "; unset UV_SYSTEM_PYTHON; uv sync" in prepare_command + + +@pytest.mark.parametrize("harness_id", [None, "rlm", "codex"]) +def test_openenv_uses_an_external_mcp_harness(harness_id: str | None) -> None: + harness: dict[str, object] = {"runtime": {"type": "docker"}} + if harness_id: + harness["id"] = harness_id + config = EnvConfig.model_validate( + { + "taskset": { + "id": "openenv", + "image": "openenv:test", + "prompt": "Use the available tool.", + "resources": {"cpu": 2, "memory": 4, "disk": 10}, + }, + "harness": harness, + } + ) + env = Environment(config) + task = env.taskset.load_tasks()[0] + toolset = env.taskset.tools(task)[0] + + assert env.harness.config.id == (harness_id or "default") + assert OpenEnvConfig.model_fields["image"].is_required() + assert OpenEnvConfig.model_fields["prompt"].is_required() + assert task.name == "openenv" + assert task.image == "openenv:test" + assert task.workdir == "/app/env" + assert task.resources.cpu == 2 + assert task.resources.memory == 4 + assert task.resources.disk == 10 + assert isinstance(toolset, vf.JSONRPCToolset) + assert toolset.config.colocated is True + assert toolset.config.endpoint == f"{OPENENV_URL}/mcp" + assert toolset.config.uds == OPENENV_SOCKET + + +def test_echo_image_is_owned_by_the_example(monkeypatch) -> None: + monkeypatch.syspath_prepend(str(ECHO_EXAMPLE)) + from openenv_echo_v1.taskset import ( + ECHO_IMAGE, + OpenEnvEchoConfig, + OpenEnvEchoTaskset, + ) + + taskset = OpenEnvEchoTaskset(OpenEnvEchoConfig(id="openenv-echo-v1")) + task = taskset.load_tasks()[0] + + assert task.name == "openenv-echo-v1" + assert task.image == ECHO_IMAGE + assert task.prompt == ( + 'Call the echo_message tool with the message "Hello, World!", then return ' + "the echoed text." + ) + assert task.resources.cpu == 2 + assert task.resources.memory == 4 + assert task.resources.disk == 10 + + +@pytest.mark.asyncio +async def test_openenv_setup_only_starts_its_server() -> None: + taskset = OpenEnvTaskset( + OpenEnvConfig(id="openenv", image="openenv:test", prompt="Use the tool.") + ) + runtime = AsyncMock() + + await taskset.setup(taskset.load_tasks()[0], runtime) + + runtime.run_background.assert_awaited_once_with( + [ + "/app/.venv/bin/uvicorn", + "server.app:app", + "--uds", + OPENENV_SOCKET, + ], + {"ENABLE_WEB_INTERFACE": "false"}, + "/tmp/openenv.log", + ) + + +@pytest.mark.asyncio +async def test_codex_harness_receives_openenv_mcp_server() -> None: + from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig + + runtime = Mock() + runtime.run_program = AsyncMock( + return_value=ProgramResult(exit_code=0, stdout="", stderr="") + ) + harness = CodexHarness(CodexHarnessConfig(id="codex")) + + await harness.launch( + vf.RolloutContext( + model="nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", + client=Mock(), + sampling=vf.SamplingConfig(), + ), + vf.Trace(task=vf.Task(idx=0, prompt="Use the Echo tool.")), + runtime, + endpoint="http://127.0.0.1:9000/v1", + secret="secret", + mcp_urls={"jsonrpc": "http://127.0.0.1:12345/mcp"}, + ) + + argv, _ = runtime.run_program.await_args.args + assert CodexHarness.SUPPORTS_MCP is True + assert harness.config.version == "0.116.0" + assert 'mcp_servers.jsonrpc.url="http://127.0.0.1:12345/mcp"' in argv + + +@pytest.mark.asyncio +async def test_jsonrpc_toolset_bridges_tools_to_mcp(monkeypatch) -> None: + responses = iter( + [ + { + "jsonrpc": "2.0", + "id": 1, + "result": { + "tools": [ + { + "name": "echo", + "description": "Echo text.", + "inputSchema": { + "type": "object", + "properties": {"message": {"type": "string"}}, + "required": ["message"], + }, + } + ] + }, + }, + {"jsonrpc": "2.0", "id": 1, "result": {"data": "hello"}}, + ] + ) + requests = [] + + class Client: + def __init__(self, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + async def post(self, url, json): + requests.append((url, json)) + return httpx.Response( + 200, + json=next(responses), + request=httpx.Request("POST", url), + ) + + monkeypatch.setattr("verifiers.v1.mcp.toolset.httpx.AsyncClient", Client) + toolset = vf.JSONRPCToolset(vf.JSONRPCToolsetConfig(endpoint="http://openenv/mcp")) + await toolset.setup() + + mcp = FastMCP("test") + toolset._register(mcp) + registered = await mcp.list_tools() + result = await mcp.call_tool("echo", {"message": "hello"}) + + assert registered[0].inputSchema["required"] == ["message"] + assert "hello" in result[0].text + assert requests[-1][1]["params"] == { + "name": "echo", + "arguments": {"message": "hello"}, + } diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index bc44b488da..14197c2e5b 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -470,6 +470,10 @@ Two setup hooks, plus the shared state: A taskset exposes a task's tools via `tools(task) -> list[vf.Toolset]`, constructing each from a config field so placement is CLI-tunable (below). +For an existing JSON-RPC endpoint that exposes `tools/list` and `tools/call` but is not a +streamable-HTTP MCP server, return a `vf.JSONRPCToolset`. It discovers the remote schemas and +re-exposes the calls as ordinary MCP tools; `uds` optionally connects through a Unix socket. + ## User simulators A user simulator is a `vf.User[ConfigT]` (or `[ConfigT, StateT]`) with one hook: diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md index f2d24586c2..86ed6f0856 100644 --- a/verifiers/v1/README.md +++ b/verifiers/v1/README.md @@ -82,6 +82,7 @@ Taskset examples (the `*_v1` packages under `environments/`): | `wiki-search-v1` | a **shared** tool server (built once for the eval) + an LLM judge | | `deepwiki-v1` | an **existing remote** tool server, by URL | | `wordle-v1` | configuring the vendored `textarena` integration | +| `openenv-echo-v1` | a thin config over the reusable OpenEnv image taskset | Harness examples (under `environments/`): @@ -112,6 +113,28 @@ uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docke uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex # the codex CLI agent ``` +### OpenEnv + +The built-in `OpenEnvTaskset` maps a configured image, prompt, workdir, and resources onto an +ordinary task. It starts the image's OpenEnv server and exposes its JSON-RPC tools through +`vf.JSONRPCToolset`; the selected Verifiers harness owns the agent loop, OpenEnv owns tool +execution, and the ordinary model/tool messages form the trace. It deliberately has no default +image or OpenEnv-specific harness. + +`openenv-echo-v1` is the example environment that pins OpenEnv's official Echo image and its +hello-world prompt: + +```bash +uv run --with-editable . --with-editable environments/openenv_echo_v1 \ + eval openenv-echo-v1 -n 1 --harness.runtime.type docker +uv run --with-editable . --with-editable environments/openenv_echo_v1 \ + eval openenv-echo-v1 -n 1 --harness.runtime.type docker --harness.id rlm +``` + +Use any harness with `SUPPORTS_MCP`. Echo's production MCP contract is intentionally unscored, +so this smoke test returns a neutral `0.0` reward while retaining the hello-world tool call and +result in the Verifiers trace. + ### Swappable runtime Where code runs, behind one `Runtime` contract — the same contract backs the harness @@ -133,9 +156,8 @@ guaranteed cleanup of its resources, even on exit/interrupt. A taskset may expose task-specific tools beyond the tools shipping natively with the harness as MCP servers. Its placement (separate runtime or colocated with harness) is configurable on `taskset.tools` and reachability is handled resolved -automatically. Tools only run under a harness with `SUPPORTS_MCP` (the `default` -harness has it; `rlm` doesn't) — an incompatible pairing is refused at load. The tool examples -each show one placement: +automatically. Tools only run under a harness with `SUPPORTS_MCP` (including `default`, `codex`, +and `rlm`) — an incompatible pairing is refused at load. The tool examples each show one placement: ```bash uv run eval glossary-v1 -n 1 # colocated — in the harness's own runtime, localhost (default) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 40e955af1c..97565f92e0 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -63,6 +63,8 @@ from verifiers.v1.task import Task, TaskResources, TaskTimeout, WireTask from verifiers.v1.taskset import Taskset, TasksetConfig from verifiers.v1.mcp import ( + JSONRPCToolset, + JSONRPCToolsetConfig, Toolset, ToolsetConfig, User, @@ -196,6 +198,8 @@ "JudgeSamplingConfig", "JudgeResponse", # mcp + "JSONRPCToolset", + "JSONRPCToolsetConfig", "Toolset", "ToolsetConfig", # user simulator diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index f6f742035a..16c78b00b3 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -2,11 +2,12 @@ Codex only speaks the streaming OpenAI Responses API, so it reaches the interception server as a custom model provider (`wire_api = responses`) pointed at the rollout endpoint — served by the -Responses dialect + SSE relay (see `verifiers.v1.dialects.responses`). The binary is the static -musl release, so it drops into any linux container with no runtime deps; its bearer token (the -session secret) is read from an env var. +Responses dialect + SSE relay (see `verifiers.v1.dialects.responses`). Task MCP servers are passed +as Codex MCP config. The binary is the static musl release, so it drops into any linux container +with no runtime deps; its bearer token (the session secret) is read from an env var. """ +import json import logging import shlex @@ -39,13 +40,14 @@ class CodexHarnessConfig(HarnessConfig): """The Codex CLI harness — which codex release to install in the runtime.""" - version: str = "0.137.0" - """Codex release to install (the `rust-v` GitHub release); pinned for reproducibility.""" + version: str = "0.116.0" + """Codex release to install. The last release before custom-provider MCP namespace calls + regressed in 0.117.""" class CodexHarness(Harness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO - SUPPORTS_MCP = False # TODO + SUPPORTS_MCP = True async def setup(self, runtime: Runtime) -> None: logger.info("codex: ensuring codex %s is installed", self.config.version) @@ -83,6 +85,11 @@ async def launch( for tool in self.config.disabled_tools or [] for arg in ("--disable", tool) ] + mcp_config = [ + arg + for name, url in mcp_urls.items() + for arg in ("-c", f"mcp_servers.{name}.url={json.dumps(url)}") + ] # `-c` values parse as TOML, falling back to a raw string (so the url / `responses` # come through literally); `requires_openai_auth=false` parses as a bool. argv = [ @@ -104,6 +111,7 @@ async def launch( f"model_providers.{PROVIDER}.wire_api=responses", "-c", f"model_providers.{PROVIDER}.requires_openai_auth=false", + *mcp_config, *tool_config, prompt, ] diff --git a/verifiers/v1/mcp/__init__.py b/verifiers/v1/mcp/__init__.py index 112d55a113..fc4c424178 100644 --- a/verifiers/v1/mcp/__init__.py +++ b/verifiers/v1/mcp/__init__.py @@ -16,11 +16,18 @@ serve_user, ) from verifiers.v1.mcp.server import ServerBase -from verifiers.v1.mcp.toolset import Toolset, ToolsetConfig +from verifiers.v1.mcp.toolset import ( + JSONRPCToolset, + JSONRPCToolsetConfig, + Toolset, + ToolsetConfig, +) from verifiers.v1.mcp.user import User, UserConfig __all__ = [ "ServerBase", + "JSONRPCToolset", + "JSONRPCToolsetConfig", "Toolset", "ToolsetConfig", "User", diff --git a/verifiers/v1/mcp/toolset.py b/verifiers/v1/mcp/toolset.py index a6bf36315c..f64da02eb0 100644 --- a/verifiers/v1/mcp/toolset.py +++ b/verifiers/v1/mcp/toolset.py @@ -6,9 +6,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import asyncio +import inspect +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, Any -from pydantic import model_validator +import httpx +from pydantic import WithJsonSchema, model_validator from pydantic_config import BaseConfig from verifiers.v1.decorators import discover_decorated @@ -79,6 +83,14 @@ def _validate_placement(self) -> "ToolsetConfig": return self +class JSONRPCToolsetConfig(ToolsetConfig): + """A JSON-RPC endpoint exposing ``tools/list`` and ``tools/call``.""" + + endpoint: str + uds: str | None = None + """Optional Unix socket used to reach the endpoint.""" + + class Toolset(ServerBase[ConfigT, StateT]): """A tool server authored as a class: write `@vf.tool` methods (the model calls them as `_`; the docstring is the tool description), reading config off `self.config` and @@ -103,3 +115,94 @@ def _register(self, mcp: FastMCP) -> None: name=getattr(fn, "tool_name", None) or fn.__name__, description=(fn.__doc__ or "").strip() or None, ) + + +class JSONRPCToolset(Toolset[JSONRPCToolsetConfig]): + """Expose a JSON-RPC tool endpoint as a standard MCP server.""" + + TOOL_PREFIX = "jsonrpc" + + async def setup(self) -> None: + if self.config.uds: + async with asyncio.timeout(60): + while not Path(self.config.uds).exists(): + await asyncio.sleep(0.1) + transport = ( + httpx.AsyncHTTPTransport(uds=self.config.uds, retries=10) + if self.config.uds + else None + ) + async with httpx.AsyncClient(transport=transport, timeout=30) as client: + response = await client.post( + self.config.endpoint, + json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, + ) + response.raise_for_status() + payload = response.json() + if error := payload.get("error"): + raise RuntimeError(error.get("message", str(error))) + self.tools = payload["result"]["tools"] + + def _register(self, mcp: FastMCP) -> None: + for tool in self.tools: + + async def call(_name=tool["name"], **arguments): + transport = ( + httpx.AsyncHTTPTransport(uds=self.config.uds) + if self.config.uds + else None + ) + async with httpx.AsyncClient(transport=transport, timeout=30) as client: + response = await client.post( + self.config.endpoint, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": _name, + "arguments": { + key: value + for key, value in arguments.items() + if value is not None + }, + }, + }, + ) + response.raise_for_status() + payload = response.json() + if error := payload.get("error"): + raise RuntimeError(error.get("message", str(error))) + return payload.get("result") + + schema = ( + tool.get("inputSchema") + or tool.get("input_schema") + or {"type": "object", "properties": {}} + ) + required = set(schema.get("required", [])) + call.__name__ = tool["name"] + call.__doc__ = tool.get("description", "") + call.__signature__ = inspect.Signature( # type: ignore[attr-defined] + [ + inspect.Parameter( + name, + inspect.Parameter.KEYWORD_ONLY, + default=( + inspect.Parameter.empty + if name in required + else spec.get("default") + ), + annotation=Annotated[Any, WithJsonSchema(spec)], + ) + for name, spec in schema.get("properties", {}).items() + ], + return_annotation=Any, + ) + mcp.add_tool( + call, name=tool["name"], description=tool.get("description") or None + ) + + +if __name__ == "__main__": + JSONRPCToolset.run() diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 08bb038666..5b74d8260c 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -192,7 +192,8 @@ async def prepare_uv_script( {}, ) command = ( - f"{_ENSURE_UV}; uv sync --script {shlex.quote(path)} -q " + f"{_ENSURE_UV}; unset UV_SYSTEM_PYTHON; " + f"uv sync --script {shlex.quote(path)} -q " f"&& uv python find --script {shlex.quote(path)}" ) result = await self.run(["sh", "-c", command], env or {}) diff --git a/verifiers/v1/tasksets/__init__.py b/verifiers/v1/tasksets/__init__.py index 956f606325..bf809cecf8 100644 --- a/verifiers/v1/tasksets/__init__.py +++ b/verifiers/v1/tasksets/__init__.py @@ -5,5 +5,11 @@ `verifiers.v1.tasksets.textarena`.""" from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset +from verifiers.v1.tasksets.openenv import OpenEnvConfig, OpenEnvTaskset -__all__ = ["HarborConfig", "HarborTaskset"] +__all__ = [ + "HarborConfig", + "HarborTaskset", + "OpenEnvConfig", + "OpenEnvTaskset", +] diff --git a/verifiers/v1/tasksets/openenv/__init__.py b/verifiers/v1/tasksets/openenv/__init__.py new file mode 100644 index 0000000000..5d179a68f7 --- /dev/null +++ b/verifiers/v1/tasksets/openenv/__init__.py @@ -0,0 +1,3 @@ +from verifiers.v1.tasksets.openenv.taskset import OpenEnvConfig, OpenEnvTaskset + +__all__ = ["OpenEnvConfig", "OpenEnvTaskset"] diff --git a/verifiers/v1/tasksets/openenv/taskset.py b/verifiers/v1/tasksets/openenv/taskset.py new file mode 100644 index 0000000000..7e46b3baaa --- /dev/null +++ b/verifiers/v1/tasksets/openenv/taskset.py @@ -0,0 +1,62 @@ +"""Run a prebuilt OpenEnv MCP image as a Verifiers taskset.""" + +from verifiers.v1.mcp import JSONRPCToolset, JSONRPCToolsetConfig, Toolset +from verifiers.v1.runtimes import Runtime +from verifiers.v1.task import Task, TaskResources +from verifiers.v1.taskset import Taskset, TasksetConfig + +OPENENV_SOCKET = "/tmp/openenv.sock" +OPENENV_URL = "http://openenv" + + +class OpenEnvConfig(TasksetConfig): + image: str + """Prebuilt OpenEnv MCP image.""" + prompt: str + """Opening prompt for the agent.""" + workdir: str = "/app/env" + """Environment project directory inside the image.""" + app: str = "server.app:app" + """ASGI app from the image's OpenEnv manifest.""" + system_prompt: str | None = None + resources: TaskResources = TaskResources() + + +class OpenEnvTaskset(Taskset[Task, OpenEnvConfig]): + NEEDS_CONTAINER = True + + def load_tasks(self) -> list[Task]: + return [ + Task( + idx=0, + name=self.config.name, + prompt=self.config.prompt, + system_prompt=self.config.system_prompt, + image=self.config.image, + workdir=self.config.workdir, + resources=self.config.resources, + ) + ] + + async def setup(self, task: Task, runtime: Runtime) -> None: + await runtime.run_background( + [ + "/app/.venv/bin/uvicorn", + self.config.app, + "--uds", + OPENENV_SOCKET, + ], + {"ENABLE_WEB_INTERFACE": "false"}, + "/tmp/openenv.log", + ) + + def tools(self, task: Task) -> list[Toolset]: + return [ + JSONRPCToolset( + JSONRPCToolsetConfig( + colocated=True, + endpoint=f"{OPENENV_URL}/mcp", + uds=OPENENV_SOCKET, + ) + ) + ] From 0a2178f3276dbb78e09fc47eb8ad97e14da792e1 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:51:44 +0200 Subject: [PATCH 2/3] Remove OpenEnv test file --- tests/v1/test_openenv.py | 205 --------------------------------------- 1 file changed, 205 deletions(-) delete mode 100644 tests/v1/test_openenv.py diff --git a/tests/v1/test_openenv.py b/tests/v1/test_openenv.py deleted file mode 100644 index 3bb1d95160..0000000000 --- a/tests/v1/test_openenv.py +++ /dev/null @@ -1,205 +0,0 @@ -from pathlib import Path -from unittest.mock import AsyncMock, Mock - -import httpx -import pytest -from mcp.server.fastmcp import FastMCP - -import verifiers.v1 as vf -from verifiers.v1.env import EnvConfig, Environment -from verifiers.v1.runtimes import ProgramResult -from verifiers.v1.runtimes.subprocess import SubprocessConfig, SubprocessRuntime -from verifiers.v1.tasksets.openenv import OpenEnvConfig, OpenEnvTaskset -from verifiers.v1.tasksets.openenv.taskset import ( - OPENENV_SOCKET, - OPENENV_URL, -) - -ECHO_EXAMPLE = Path(__file__).parents[2] / "environments" / "openenv_echo_v1" - - -@pytest.mark.asyncio -async def test_uv_script_ignores_image_system_python() -> None: - runtime = SubprocessRuntime(SubprocessConfig()) - runtime.write = AsyncMock() - runtime.run = AsyncMock( - side_effect=[ - ProgramResult(exit_code=0, stdout="", stderr=""), - ProgramResult( - exit_code=0, stdout="/tmp/openenv-test/bin/python\n", stderr="" - ), - ] - ) - - await runtime.prepare_uv_script("# openenv image system-python regression\n") - - prepare_command = runtime.run.await_args_list[1].args[0][2] - assert "; unset UV_SYSTEM_PYTHON; uv sync" in prepare_command - - -@pytest.mark.parametrize("harness_id", [None, "rlm", "codex"]) -def test_openenv_uses_an_external_mcp_harness(harness_id: str | None) -> None: - harness: dict[str, object] = {"runtime": {"type": "docker"}} - if harness_id: - harness["id"] = harness_id - config = EnvConfig.model_validate( - { - "taskset": { - "id": "openenv", - "image": "openenv:test", - "prompt": "Use the available tool.", - "resources": {"cpu": 2, "memory": 4, "disk": 10}, - }, - "harness": harness, - } - ) - env = Environment(config) - task = env.taskset.load_tasks()[0] - toolset = env.taskset.tools(task)[0] - - assert env.harness.config.id == (harness_id or "default") - assert OpenEnvConfig.model_fields["image"].is_required() - assert OpenEnvConfig.model_fields["prompt"].is_required() - assert task.name == "openenv" - assert task.image == "openenv:test" - assert task.workdir == "/app/env" - assert task.resources.cpu == 2 - assert task.resources.memory == 4 - assert task.resources.disk == 10 - assert isinstance(toolset, vf.JSONRPCToolset) - assert toolset.config.colocated is True - assert toolset.config.endpoint == f"{OPENENV_URL}/mcp" - assert toolset.config.uds == OPENENV_SOCKET - - -def test_echo_image_is_owned_by_the_example(monkeypatch) -> None: - monkeypatch.syspath_prepend(str(ECHO_EXAMPLE)) - from openenv_echo_v1.taskset import ( - ECHO_IMAGE, - OpenEnvEchoConfig, - OpenEnvEchoTaskset, - ) - - taskset = OpenEnvEchoTaskset(OpenEnvEchoConfig(id="openenv-echo-v1")) - task = taskset.load_tasks()[0] - - assert task.name == "openenv-echo-v1" - assert task.image == ECHO_IMAGE - assert task.prompt == ( - 'Call the echo_message tool with the message "Hello, World!", then return ' - "the echoed text." - ) - assert task.resources.cpu == 2 - assert task.resources.memory == 4 - assert task.resources.disk == 10 - - -@pytest.mark.asyncio -async def test_openenv_setup_only_starts_its_server() -> None: - taskset = OpenEnvTaskset( - OpenEnvConfig(id="openenv", image="openenv:test", prompt="Use the tool.") - ) - runtime = AsyncMock() - - await taskset.setup(taskset.load_tasks()[0], runtime) - - runtime.run_background.assert_awaited_once_with( - [ - "/app/.venv/bin/uvicorn", - "server.app:app", - "--uds", - OPENENV_SOCKET, - ], - {"ENABLE_WEB_INTERFACE": "false"}, - "/tmp/openenv.log", - ) - - -@pytest.mark.asyncio -async def test_codex_harness_receives_openenv_mcp_server() -> None: - from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig - - runtime = Mock() - runtime.run_program = AsyncMock( - return_value=ProgramResult(exit_code=0, stdout="", stderr="") - ) - harness = CodexHarness(CodexHarnessConfig(id="codex")) - - await harness.launch( - vf.RolloutContext( - model="nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", - client=Mock(), - sampling=vf.SamplingConfig(), - ), - vf.Trace(task=vf.Task(idx=0, prompt="Use the Echo tool.")), - runtime, - endpoint="http://127.0.0.1:9000/v1", - secret="secret", - mcp_urls={"jsonrpc": "http://127.0.0.1:12345/mcp"}, - ) - - argv, _ = runtime.run_program.await_args.args - assert CodexHarness.SUPPORTS_MCP is True - assert harness.config.version == "0.116.0" - assert 'mcp_servers.jsonrpc.url="http://127.0.0.1:12345/mcp"' in argv - - -@pytest.mark.asyncio -async def test_jsonrpc_toolset_bridges_tools_to_mcp(monkeypatch) -> None: - responses = iter( - [ - { - "jsonrpc": "2.0", - "id": 1, - "result": { - "tools": [ - { - "name": "echo", - "description": "Echo text.", - "inputSchema": { - "type": "object", - "properties": {"message": {"type": "string"}}, - "required": ["message"], - }, - } - ] - }, - }, - {"jsonrpc": "2.0", "id": 1, "result": {"data": "hello"}}, - ] - ) - requests = [] - - class Client: - def __init__(self, **kwargs): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - pass - - async def post(self, url, json): - requests.append((url, json)) - return httpx.Response( - 200, - json=next(responses), - request=httpx.Request("POST", url), - ) - - monkeypatch.setattr("verifiers.v1.mcp.toolset.httpx.AsyncClient", Client) - toolset = vf.JSONRPCToolset(vf.JSONRPCToolsetConfig(endpoint="http://openenv/mcp")) - await toolset.setup() - - mcp = FastMCP("test") - toolset._register(mcp) - registered = await mcp.list_tools() - result = await mcp.call_tool("echo", {"message": "hello"}) - - assert registered[0].inputSchema["required"] == ["message"] - assert "hello" in result[0].text - assert requests[-1][1]["params"] == { - "name": "echo", - "arguments": {"message": "hello"}, - } From e96a9f1ceda91bd46782cebb14caea63fc4a7e6c Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Mon, 29 Jun 2026 12:54:49 +0200 Subject: [PATCH 3/3] Add OpenEnv gym contract support --- verifiers/v1/README.md | 18 ++- verifiers/v1/env.py | 11 +- verifiers/v1/taskset.py | 8 + verifiers/v1/tasksets/openenv/taskset.py | 193 ++++++++++++++++++++++- 4 files changed, 205 insertions(+), 25 deletions(-) diff --git a/verifiers/v1/README.md b/verifiers/v1/README.md index 86ed6f0856..5f6041f36e 100644 --- a/verifiers/v1/README.md +++ b/verifiers/v1/README.md @@ -115,11 +115,14 @@ uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docke ### OpenEnv -The built-in `OpenEnvTaskset` maps a configured image, prompt, workdir, and resources onto an -ordinary task. It starts the image's OpenEnv server and exposes its JSON-RPC tools through -`vf.JSONRPCToolset`; the selected Verifiers harness owns the agent loop, OpenEnv owns tool -execution, and the ordinary model/tool messages form the trace. It deliberately has no default -image or OpenEnv-specific harness. +The built-in `OpenEnvTaskset` maps a configured image, OpenEnv contract, workdir, and resources +onto an ordinary task. It deliberately has no default image or OpenEnv-specific harness. + +- `contract = "mcp"` starts the image's OpenEnv server and exposes its JSON-RPC tools through + `vf.JSONRPCToolset`; use any harness with `SUPPORTS_MCP`. +- `contract = "gym"` starts the same server, resets it over OpenEnv's persistent WebSocket API, + renders observations as user turns, and sends assistant JSON actions into `step`; use a harness + with `SUPPORTS_USER_SIM`. `openenv-echo-v1` is the example environment that pins OpenEnv's official Echo image and its hello-world prompt: @@ -131,9 +134,8 @@ uv run --with-editable . --with-editable environments/openenv_echo_v1 \ eval openenv-echo-v1 -n 1 --harness.runtime.type docker --harness.id rlm ``` -Use any harness with `SUPPORTS_MCP`. Echo's production MCP contract is intentionally unscored, -so this smoke test returns a neutral `0.0` reward while retaining the hello-world tool call and -result in the Verifiers trace. +Echo's production MCP contract is intentionally unscored, so this smoke test returns a neutral +`0.0` reward while retaining the hello-world tool call and result in the Verifiers trace. ### Swappable runtime diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 7113b2af81..f1623208ed 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -231,24 +231,17 @@ def resolve_runtime_config( class Environment: def __init__(self, config: EnvConfig) -> None: from verifiers.v1.loaders import load_harness, load_taskset - from verifiers.v1.taskset import Taskset self.config = config self.taskset = load_taskset(config.taskset) self.harness = load_harness(config.harness) - if ( - not self.harness.SUPPORTS_MCP - and type(self.taskset).tools is not Taskset.tools - ): + if not self.harness.SUPPORTS_MCP and self.taskset.has_tools(): raise ValueError( f"Harness {self.harness.config.id!r} does not support MCP tools, but taskset " f"{self.taskset.config.id!r} exposes tool servers (MCP). Run it with a harness " f"that supports MCP (e.g. --harness.id default), or use a taskset without tools." ) - if ( - not self.harness.SUPPORTS_USER_SIM - and type(self.taskset).user is not Taskset.user - ): + if not self.harness.SUPPORTS_USER_SIM and self.taskset.has_user(): raise ValueError( f"Harness {self.harness.config.id!r} does not drive a user simulator, but taskset " f"{self.taskset.config.id!r} defines one (Taskset.user). Run it with a harness that " diff --git a/verifiers/v1/taskset.py b/verifiers/v1/taskset.py index 716a23cfd8..d88cccb8bf 100644 --- a/verifiers/v1/taskset.py +++ b/verifiers/v1/taskset.py @@ -72,6 +72,10 @@ def tools(self, task: TaskT) -> list[Toolset]: for an already-running server). Empty by default; override to give a task tools.""" return [] + def has_tools(self) -> bool: + """Whether this configured taskset can expose MCP tools.""" + return type(self).tools is not Taskset.tools + def user(self, task: TaskT) -> User | None: """A user simulator for this task — structurally a tool server (an MCP server with a runtime), but driven by the framework, not exposed to the model. After @@ -80,6 +84,10 @@ def user(self, task: TaskT) -> User | None: multi-turn conversation (e.g. a TextArena game).""" return None + def has_user(self) -> bool: + """Whether this configured taskset can drive the rollout through a user simulator.""" + return type(self).user is not Taskset.user + async def setup(self, task: TaskT, runtime: Runtime) -> None: """Prepare the live runtime for this task, after `runtime.start()` and before the harness runs. No-op by default; override to run per-task setup in the runtime (e.g. diff --git a/verifiers/v1/tasksets/openenv/taskset.py b/verifiers/v1/tasksets/openenv/taskset.py index 7e46b3baaa..5e6832daa9 100644 --- a/verifiers/v1/tasksets/openenv/taskset.py +++ b/verifiers/v1/tasksets/openenv/taskset.py @@ -1,36 +1,188 @@ -"""Run a prebuilt OpenEnv MCP image as a Verifiers taskset.""" +"""Run a prebuilt OpenEnv image as a Verifiers taskset.""" -from verifiers.v1.mcp import JSONRPCToolset, JSONRPCToolsetConfig, Toolset +from __future__ import annotations + +import json +from typing import Any, Literal + +import aiohttp +from pydantic import Field + +from verifiers.v1.decorators import reward, stop +from verifiers.v1.mcp import ( + JSONRPCToolset, + JSONRPCToolsetConfig, + Toolset, + User, + UserConfig, +) from verifiers.v1.runtimes import Runtime +from verifiers.v1.state import State from verifiers.v1.task import Task, TaskResources from verifiers.v1.taskset import Taskset, TasksetConfig +from verifiers.v1.types import Messages OPENENV_SOCKET = "/tmp/openenv.sock" OPENENV_URL = "http://openenv" +ACTION_INSTRUCTIONS = ( + "Respond with only a JSON object matching this OpenEnv action schema. " + "If the schema has one required string field, you may reply with just that string value." +) + + +class OpenEnvState(State): + done: bool = False + reward: float = 0.0 + action_schema: dict[str, Any] = Field(default_factory=dict) class OpenEnvConfig(TasksetConfig): - image: str - """Prebuilt OpenEnv MCP image.""" - prompt: str - """Opening prompt for the agent.""" + image: str | None = None + """Prebuilt OpenEnv image.""" + contract: Literal["mcp", "gym"] = "mcp" + """OpenEnv runtime contract exposed by the image.""" + prompt: str | None = None + """Opening prompt for MCP environments. Gym environments open from reset().""" workdir: str = "/app/env" """Environment project directory inside the image.""" app: str = "server.app:app" """ASGI app from the image's OpenEnv manifest.""" system_prompt: str | None = None resources: TaskResources = TaskResources() + reset: dict[str, Any] = Field(default_factory=dict) + """Extra reset kwargs for gym environments.""" + + +class OpenEnvUserConfig(UserConfig): + colocated: bool = True + base_url: str = OPENENV_URL + uds: str = OPENENV_SOCKET + reset: dict[str, Any] = Field(default_factory=dict) + timeout: float = 60.0 + + +class OpenEnvUser(User[OpenEnvUserConfig, OpenEnvState]): + async def setup_task(self, task) -> None: + self.session: aiohttp.ClientSession | None = None + self.ws: aiohttp.ClientWebSocketResponse | None = None + self.action_schema: dict[str, Any] = {} + self.initial: dict[str, Any] | None = None + + async def respond(self, message: str) -> Messages: + await self._connect() + if message: + result = await self._send( + {"type": "step", "data": self._parse_action(message)} + ) + else: + if self.initial is None: + self.initial = await self._send( + {"type": "reset", "data": self.config.reset} + ) + result = self.initial + + data = result.get("data", {}) + self.state.done = bool(data.get("done", False)) + self.state.reward += float(data.get("reward") or 0.0) + self.state.action_schema = self.action_schema + if self.state.done: + await self._close() + return self._render(data.get("observation", {})) + + async def _connect(self) -> None: + if self.ws is not None: + return + connector = aiohttp.UnixConnector(path=self.config.uds) + timeout = aiohttp.ClientTimeout(total=self.config.timeout) + self.session = aiohttp.ClientSession(connector=connector, timeout=timeout) + self.ws = await self.session.ws_connect(f"{self.config.base_url}/ws") + self.action_schema = await self._schema() + async def _close(self) -> None: + if self.ws is not None: + await self.ws.close() + self.ws = None + if self.session is not None: + await self.session.close() + self.session = None -class OpenEnvTaskset(Taskset[Task, OpenEnvConfig]): + async def _schema(self) -> dict[str, Any]: + if self.session is None: + raise RuntimeError("OpenEnv gym client is not connected.") + async with self.session.get(f"{self.config.base_url}/schema") as response: + response.raise_for_status() + payload = await response.json() + schema = payload.get("action", {}) + return schema if isinstance(schema, dict) else {} + + async def _send(self, payload: dict[str, Any]) -> dict[str, Any]: + if self.ws is None: + raise RuntimeError("OpenEnv gym client is not connected.") + await self.ws.send_json(payload) + message = await self.ws.receive_json() + if message.get("type") == "error": + raise RuntimeError(message.get("data", {}).get("message", str(message))) + return message + + def _parse_action(self, message: str) -> dict[str, Any]: + text = message.strip() + if text.startswith("```"): + text = text.strip("`").removeprefix("json").strip() + try: + action = json.loads(text) + if isinstance(action, dict): + return action + except json.JSONDecodeError: + pass + + properties = self.action_schema.get("properties", {}) + required = self.action_schema.get("required", []) + if ( + isinstance(properties, dict) + and isinstance(required, list) + and len(required) == 1 + and properties.get(required[0], {}).get("type") == "string" + ): + return {required[0]: text} + raise ValueError( + "OpenEnv gym actions must be JSON objects matching the action schema." + ) + + def _render(self, observation: object) -> Messages: + schema = json.dumps(self.action_schema, ensure_ascii=True) + suffix = f"\n\n{ACTION_INSTRUCTIONS}\n{schema}" + if isinstance(observation, dict): + messages = observation.get("messages") + if isinstance(messages, list) and messages: + rendered = [ + {"role": "user", "content": str(item.get("content", item))} + if isinstance(item, dict) + else {"role": "user", "content": str(item)} + for item in messages + ] + rendered[-1]["content"] += suffix + return rendered + for key in ("prompt", "question", "instruction", "content", "text"): + value = observation.get(key) + if isinstance(value, str) and value.strip(): + return [{"role": "user", "content": f"{value.strip()}{suffix}"}] + content = json.dumps(observation, ensure_ascii=True, default=str) + return [{"role": "user", "content": f"{content}{suffix}"}] + + +class OpenEnvTaskset(Taskset[Task, OpenEnvConfig, OpenEnvState]): NEEDS_CONTAINER = True def load_tasks(self) -> list[Task]: + if self.config.image is None: + raise ValueError("OpenEnv tasksets require `image`.") + if self.config.contract == "mcp" and self.config.prompt is None: + raise ValueError("OpenEnv MCP tasksets require `prompt`.") return [ Task( idx=0, name=self.config.name, - prompt=self.config.prompt, + prompt=self.config.prompt if self.config.contract == "mcp" else None, system_prompt=self.config.system_prompt, image=self.config.image, workdir=self.config.workdir, @@ -50,7 +202,12 @@ async def setup(self, task: Task, runtime: Runtime) -> None: "/tmp/openenv.log", ) + def has_tools(self) -> bool: + return self.config.contract == "mcp" + def tools(self, task: Task) -> list[Toolset]: + if self.config.contract != "mcp": + return [] return [ JSONRPCToolset( JSONRPCToolsetConfig( @@ -60,3 +217,23 @@ def tools(self, task: Task) -> list[Toolset]: ) ) ] + + def has_user(self) -> bool: + return self.config.contract == "gym" + + def user(self, task: Task) -> User | None: + if self.config.contract != "gym": + return None + return OpenEnvUser(OpenEnvUserConfig(reset=self.config.reset)) + + @stop + async def openenv_done(self, trace) -> bool: + return trace.state.done + + @reward(weight=1.0) + async def openenv_reward(self, trace) -> float: + return trace.state.reward + + +if __name__ == "__main__": + OpenEnvUser.run()