Add NeMo Gym

environments/README.md

@@ -59,7 +59,6 @@ This folder contains installable example environments that showcase common usage
   - **bfcl_v3**: BFCL v3 function-calling eval using task-local dynamic tool schemas and v1 rewards.
   - **dspy_flights**: Sandboxed DSPy flight-support `program.fn` entrypoint installed from its package `pyproject.toml` and configured against the v1 interception endpoint.
   - **hello_group_reward_v1**: Deterministic v1 reference for group updates, metrics, rewards, advantages, and cleanup.
-  - **nemo_gym_env**: Minimal v1 example that wraps a packaged NeMo Gym task with `NeMoGymTaskset` and `NeMoGymHarness`.
   - **sft-replay**: Thin v1 replay environment using `ReplayTaskset` and `ReplayHarness` to turn stored transcripts into trajectory steps without model calls.
   - **wordle_v1**: TextArena Wordle through the packaged v1 `TextArenaTaskset` boundary.
 

tests/v1/test_nemo_gym.py

@@ -0,0 +1,107 @@
+import json
+import sys
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from verifiers.v1.env import EnvConfig, Environment
+from verifiers.v1.loaders import default_harness_id, taskset_config_type
+from verifiers.v1.tasksets.nemo_gym import (
+    NeMoGymConfig,
+    NeMoGymTask,
+    NeMoGymTaskset,
+)
+from verifiers.v1.types import SystemMessage, UserMessage
+
+
+def test_nemo_gym_loads_packaged_example(tmp_path, monkeypatch) -> None:
+    data = (
+        tmp_path
+        / "resources_servers"
+        / "example_single_tool_call"
+        / "data"
+        / "example.jsonl"
+    )
+    data.parent.mkdir(parents=True)
+    row = {
+        "responses_create_params": {
+            "input": [
+                {"role": "developer", "content": "Be helpful."},
+                {"role": "user", "content": "What's it like in SF?"},
+            ],
+            "tools": [
+                {
+                    "type": "function",
+                    "name": "get_weather",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"city": {"type": "string"}},
+                        "required": ["city"],
+                    },
+                }
+            ],
+        }
+    }
+    data.write_text(json.dumps(row) + "\n")
+    monkeypatch.setitem(sys.modules, "nemo_gym", SimpleNamespace(PARENT_DIR=tmp_path))
+
+    [task] = NeMoGymTaskset(NeMoGymConfig()).load_tasks()
+
+    assert task.name == "example_single_tool_call:0"
+    assert task.prompt == [
+        SystemMessage(content="Be helpful."),
+        UserMessage(content="What's it like in SF?"),
+    ]
+    assert "nemo_gym_call" in task.system_prompt
+    assert "get_weather" in task.system_prompt
+    assert task.nemo_gym_row == row
+
+
+def test_nemo_gym_uses_the_standard_verifiers_harness() -> None:
+    env = Environment(
+        EnvConfig.model_validate(
+            {
+                "taskset": {"id": "nemo_gym"},
+                "harness": {},
+            }
+        )
+    )
+    task = NeMoGymTask(
+        idx=0,
+        prompt="hello",
+        nemo_gym_row={"responses_create_params": {"tools": [{"name": "tool"}]}},
+    )
+
+    [toolset] = env.taskset.tools(task)
+
+    assert env.harness.config.id == "default"
+    assert toolset.server_name == "nemo_gym"
+    assert toolset.config.nemo_env == "example_single_tool_call"
+
+
+@pytest.mark.asyncio
+async def test_nemo_gym_forwards_tools_to_the_resource_server() -> None:
+    taskset = NeMoGymTaskset(NeMoGymConfig())
+    task = NeMoGymTask(
+        idx=0,
+        prompt="hello",
+        nemo_gym_row={"responses_create_params": {"tools": [{"name": "get_weather"}]}},
+    )
+    [toolset] = taskset.tools(task)
+    toolset.tool_names = {"get_weather"}
+    response = Mock()
+    response.json.return_value = {"weather_description": "cold"}
+    toolset.client = AsyncMock()
+    toolset.client.post.return_value = response
+
+    result = await toolset.call("get_weather", {"city": "sf"})
+
+    toolset.client.post.assert_awaited_once_with("/get_weather", json={"city": "sf"})
+    response.raise_for_status.assert_called_once()
+    assert result == {"weather_description": "cold"}
+
+
+def test_nemo_gym_is_only_a_taskset() -> None:
+    assert default_harness_id("nemo_gym") == "default"
+    assert taskset_config_type("nemo_gym") is NeMoGymConfig

verifiers/v1/README.md

@@ -112,6 +112,22 @@ uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docke
 uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex           # the codex CLI agent
 ```
 
+### NeMo Gym
+
+The built-in `nemo_gym` taskset defaults to NeMo Gym's own
+`example_single_tool_call` quickstart. It loads the example JSONL as ordinary typed tasks;
+the selected Verifiers harness owns the agent loop, while NeMo Gym's packaged resource
+server executes the declared tools. Model and tool messages use the normal Verifiers trace.
+There is no NeMo-specific harness or program.
+
+```bash
+uv run --with nemo-gym==0.3.0 eval nemo_gym -n 1 -r 1 -c 1
+```
+
+The taskset does not interpret the example's weather behavior. It forwards declared calls
+to the corresponding NeMo Gym resource server through one generic `nemo_gym_call` tool.
+The standard Verifiers harness drives the model/tool loop and captures the trace.
+
 ### Swappable runtime
 
 Where code runs, behind one `Runtime` contract — the same contract backs the harness

verifiers/v1/tasksets/__init__.py

@@ -5,5 +5,11 @@
 `verifiers.v1.tasksets.textarena`."""
 
 from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
+from verifiers.v1.tasksets.nemo_gym import NeMoGymConfig, NeMoGymTaskset
 
-__all__ = ["HarborConfig", "HarborTaskset"]
+__all__ = [
+    "HarborConfig",
+    "HarborTaskset",
+    "NeMoGymConfig",
+    "NeMoGymTaskset",
+]

verifiers/v1/tasksets/nemo_gym/__init__.py

@@ -0,0 +1,11 @@
+from verifiers.v1.tasksets.nemo_gym.taskset import (
+    NeMoGymConfig,
+    NeMoGymTask,
+    NeMoGymTaskset,
+)
+
+__all__ = [
+    "NeMoGymConfig",
+    "NeMoGymTask",
+    "NeMoGymTaskset",
+]

verifiers/v1/tasksets/nemo_gym/taskset.py

@@ -0,0 +1,157 @@
+"""Run a packaged NeMo Gym dataset with any MCP-capable Verifiers harness."""
+
+import importlib
+import json
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from verifiers.v1.decorators import tool
+from verifiers.v1.dialects.responses import ResponsesDialect
+from verifiers.v1.mcp import Toolset, ToolsetConfig
+from verifiers.v1.task import Task
+from verifiers.v1.taskset import Taskset, TasksetConfig
+
+DEFAULT_ENV = "example_single_tool_call"
+
+
+class NeMoGymTask(Task):
+    nemo_gym_row: dict[str, Any]
+
+
+class NeMoGymConfig(TasksetConfig):
+    nemo_env: str = DEFAULT_ENV
+    data_name: str = "example.jsonl"
+
+
+class _NeMoGymToolsConfig(ToolsetConfig):
+    nemo_env: str
+
+
+class _NeMoGymTools(Toolset[_NeMoGymToolsConfig]):
+    """Expose a NeMo Gym resource server through one generic MCP call."""
+
+    TOOL_PREFIX = "nemo_gym"
+
+    async def setup(self) -> None:
+        from nemo_gym import PARENT_DIR
+        from nemo_gym.base_resources_server import SimpleResourcesServer
+        from nemo_gym.config_types import BaseServerConfig
+        from nemo_gym.global_config import (
+            GlobalConfigDictParser,
+            GlobalConfigDictParserConfig,
+        )
+        from nemo_gym.server_utils import ServerClient
+        from omegaconf import OmegaConf
+
+        root = Path(PARENT_DIR)
+        path = (
+            root
+            / "resources_servers"
+            / self.config.nemo_env
+            / "configs"
+            / f"{self.config.nemo_env}.yaml"
+        )
+        parser = GlobalConfigDictParser()
+        config = parser.parse_no_environment(
+            OmegaConf.merge(
+                OmegaConf.load(path),
+                GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT,
+            )
+        )
+        resource = next(
+            server
+            for server in parser.filter_for_server_instance_configs(config)
+            if server.SERVER_TYPE == "resources_servers"
+        )
+        entrypoint = Path(resource.get_inner_run_server_config().entrypoint).stem
+        module = importlib.import_module(
+            f"resources_servers.{self.config.nemo_env}.{entrypoint}"
+        )
+        server_cls: Any = next(
+            value
+            for value in vars(module).values()
+            if isinstance(value, type)
+            and issubclass(value, SimpleResourcesServer)
+            and value.__module__ == module.__name__
+        )
+        server_config = server_cls.model_fields["config"].annotation.model_validate(
+            {
+                "name": resource.name,
+                **resource.get_inner_run_server_config().model_dump(),
+            }
+        )
+        server = server_cls(
+            config=server_config,
+            server_client=ServerClient(
+                head_server_config=BaseServerConfig.model_validate(
+                    config["head_server"]
+                ),
+                global_config_dict=config,
+            ),
+        )
+        self.client = httpx.AsyncClient(
+            transport=httpx.ASGITransport(app=server.setup_webserver()),
+            base_url="http://nemo-gym",
+        )
+
+    async def setup_task(self, task: NeMoGymTask) -> None:
+        self.tool_names = {
+            spec["name"]
+            for spec in task.nemo_gym_row["responses_create_params"].get("tools", [])
+        }
+        response = await self.client.post("/seed_session", json=task.nemo_gym_row)
+        response.raise_for_status()
+
+    @tool
+    async def call(self, name: str, arguments: dict[str, Any]) -> Any:
+        """Call a tool declared by the current NeMo Gym task."""
+        if name not in self.tool_names:
+            raise ValueError(f"unknown NeMo Gym tool: {name}")
+        response = await self.client.post(f"/{name}", json=arguments)
+        response.raise_for_status()
+        return response.json()
+
+
+class NeMoGymTaskset(Taskset[NeMoGymTask, NeMoGymConfig]):
+    def load_tasks(self) -> list[NeMoGymTask]:
+        try:
+            from nemo_gym import PARENT_DIR
+        except ImportError as exc:
+            raise ImportError(
+                "Run this taskset with `uv run --with nemo-gym==0.3.0 eval nemo_gym`."
+            ) from exc
+
+        path = (
+            Path(PARENT_DIR)
+            / "resources_servers"
+            / self.config.nemo_env
+            / "data"
+            / self.config.data_name
+        )
+        rows = [json.loads(line) for line in path.read_text().splitlines() if line]
+        dialect = ResponsesDialect()
+        return [
+            NeMoGymTask(
+                idx=idx,
+                name=f"{self.config.nemo_env}:{idx}",
+                prompt=dialect.parse_request(row["responses_create_params"])[0],
+                system_prompt=(
+                    "Call `nemo_gym_call` with the matching tool name and arguments. "
+                    f"Available NeMo Gym tools: "
+                    f"{json.dumps(row['responses_create_params'].get('tools', []))}"
+                ),
+                nemo_gym_row=row,
+            )
+            for idx, row in enumerate(rows)
+        ]
+
+    def tools(self, task: NeMoGymTask) -> list[Toolset]:
+        if not task.nemo_gym_row["responses_create_params"].get("tools"):
+            return []
+        return [_NeMoGymTools(_NeMoGymToolsConfig(nemo_env=self.config.nemo_env))]
+
+
+if __name__ == "__main__":
+    _NeMoGymTools.run()