Add LeRobot LIBERO policy rollout

CONTEXT.md

@@ -64,6 +64,90 @@ _Avoid_: independent per-robot timing, per-arm retiming when referring to coordi
 An execution-preparation artifact that derives control-task-specific joint trajectory messages from a generated trajectory without changing the generated trajectory's canonical global timing.
 _Avoid_: generated trajectory projection, execution-time parametrization, per-task generated trajectory
 
+**Policy trajectory dispatch**:
+An execution-preparation artifact that derives control-task-specific joint trajectory messages from policy action chunks while leaving control authority with the configured control task.
+_Avoid_: direct policy control, policy motor write, learned controller bypass
+
+**Joint-trajectory policy chunk**:
+A v1 robot-learning action chunk shaped to become the `JointTrajectory` argument accepted by a joint trajectory control task.
+_Avoid_: generic policy action, direct motor command, backend-native tensor
+
+**Contract-assigned trajectory timing**:
+A v1 policy rollout convention where the robot policy contract assigns simple relative timing to backend action rows when constructing a joint trajectory for dispatch, starting the first action row at `time_from_start = 0`.
+_Avoid_: controller-side chunk timing inference, untimed policy positions, trajectory parametrization
+
+**Dispatch-owned trajectory timestamp**:
+A policy rollout convention where the rollout module assigns the absolute intended start time for the whole contract-built trajectory chunk when submitting it to the control system.
+_Avoid_: contract-owned dispatch time, backend inference timestamp as control timestamp, trajectory creation time
+
+**Subchunk trajectory dispatch**:
+A policy rollout convention where the rollout module generates a larger policy action chunk but dispatches only a configured leading subset of action rows to the joint trajectory control task before replanning.
+_Avoid_: executing the entire policy horizon by default, deep policy command queue, hidden action-horizon truncation
+
+**Completion-gated policy rollout**:
+A simple rollout cadence where the policy module waits for the dispatched trajectory subset to finish before fetching the latest observations and generating the next backend action chunk.
+_Avoid_: asynchronous continuous inference, speculative trajectory queueing, wall-clock-only rollout loop
+
+**Sleep-gated policy rollout**:
+A v1 completion-gated policy rollout implementation where the policy module waits for the expected row coverage duration of the dispatched trajectory subset using the rollout clock before fetching observations and replanning.
+_Avoid_: task-completion dependency, signal-based rollout synchronization, speculative inference
+
+**Policy action representation**:
+The robot-learning action form a policy is trained or configured to produce before it is adapted for a DimOS control task.
+_Avoid_: assuming all policy actions are joint positions, opaque policy output
+
+**Robot policy action**:
+A runtime-independent robot-learning action emitted by a robot policy module after backend inference and contract conversion, before adaptation to benchmark runtime frames or real robot control commands.
+_Avoid_: runtime action frame, motor command, backend tensor
+
+**Native benchmark action surface**:
+A benchmark runtime action interface whose command values are defined by the benchmark environment itself rather than by a DimOS motor or joint surface.
+_Avoid_: motor command alias, hidden joint target, controller-specific shortcut
+
+**LIBERO action mode**:
+The simulator-side interpretation of `env.step(action)` in LIBERO, defining whether the action vector represents joint positions, relative end-effector deltas, or another environment-supported action form.
+_Avoid_: DimOS controller, ControlCoordinator task, policy output type
+
+**Native LIBERO action mode**:
+A LIBERO runtime mode, matching the official LeRobot LIBERO environment setup, that accepts the environment's own relative end-effector delta plus gripper action vector instead of a DimOS whole-body motor-position command.
+_Avoid_: joint-position LIBERO profile, fake motor command, end-effector action as motor q
+
+**Runtime action frame**:
+A runtime protocol command envelope for a named non-motor action surface, carrying action values and semantic identity without pretending they are joint positions or motor commands.
+_Avoid_: overloaded motor frame, unnamed action vector, backend tensor leak
+
+**Action-surface control task**:
+A control-coordinator task that accepts and validates commands for a semantic action surface while leaving the concrete runtime or robot mapping to the task implementation.
+_Avoid_: joint trajectory task, motor adapter, policy bypass
+
+**Robot policy contract**:
+A robot-learning boundary that declares a specific robot/runtime and policy-backend input-output convention, including how aligned robot-native samples become backend-ready batches and how backend outputs become robot-native policy action chunks.
+_Avoid_: execution contract, control-task adapter, universal robot contract
+
+**Robot learning sample**:
+A runtime-independent policy observation artifact that carries semantically named observation roles, task context, timestamps, and metadata for robot-learning inference or training.
+_Avoid_: benchmark sidecar response, runtime observation frame, backend-ready batch
+
+**Backend-ready batch**:
+The policy-backend-specific inference or training input produced from an aligned robot-native sample by a robot policy contract.
+_Avoid_: universal DimOS batch, raw observation bundle, synchronized sample
+
+**Backend output**:
+The policy-backend-specific action result that a robot policy contract converts into a robot-native policy action chunk.
+_Avoid_: control command, joint trajectory, final actuator target
+
+**Backend output envelope**:
+A small rollout artifact that carries a backend-native action result together with inference metadata needed for validation, tracing, and contract conversion.
+_Avoid_: policy action chunk, control command, backend internals leak
+
+**Temporal sample readiness**:
+The rollout-time check that the observation roles needed for a policy sample are available with acceptable freshness for the current inference tick, without requiring perfect cross-sensor timestamp equality.
+_Avoid_: exact timestamp match, strict synchronization gate, semantic policy validation
+
+**Contract conversion failure**:
+A fail-fast result when a robot policy contract receives a supposedly ready sample or backend output that violates the contract's declared semantic input-output convention.
+_Avoid_: not-ready sample, silent coercion, best-effort backend batch
+
 **Robokin kinematics backend**:
 A DimOS kinematics backend that presents multiple robokin-supported inverse-kinematics engines through one robotics-facing capability.
 _Avoid_: Oink backend, RoboKin world backend, single-engine Oink solver

dimos/benchmark/runtime/test_libero_pro_sidecar_profile.py

@@ -22,7 +22,7 @@
 from pathlib import Path
 import sys
 from threading import Thread
-from typing import cast
+from typing import Literal, cast
 from urllib.request import Request, urlopen
 
 import numpy as np
@@ -35,12 +35,20 @@
 sys.path.insert(0, str(LIBERO_PRO_SIDECAR_SRC))
 
 from dimos_libero_pro_sidecar.server import (
+    NATIVE_ACTION_SPACE_ID,
     LiberoProRuntimeConfig,
     LiberoProRuntimeState,
+    RealLiberoBackend,
+    ensure_libero_config,
     make_server,
     validate_assets,
 )
-from dimos_runtime_protocol import EpisodeResetRequest, MotorActionFrame, StepRequest
+from dimos_runtime_protocol import (
+    EpisodeResetRequest,
+    MotorActionFrame,
+    RuntimeActionFrame,
+    StepRequest,
+)
 
 
 class _FakeLiberoBackend:
@@ -75,6 +83,58 @@ class _BadActionBackend(_FakeLiberoBackend):
     action_high = [1.0] * 7
 
 
+class _FakeNativeBackend(_FakeLiberoBackend):
+    action_low = [-1.0] * 7
+    action_high = [1.0] * 7
+
+    def __init__(self) -> None:
+        self.last_action: list[float] = []
+
+    def step(
+        self, action: Sequence[float]
+    ) -> tuple[dict[str, object], float, bool, dict[str, object]]:
+        self.last_action = [float(item) for item in action]
+        return _fake_obs(self.last_action[:7], [0.0]), 0.5, False, {"success": False}
+
+
+class _BadNativeBoundsBackend(_FakeNativeBackend):
+    action_low = [-0.5] * 7
+    action_high = [1.0] * 7
+
+
+class _FakeController:
+    def __init__(self) -> None:
+        self.use_delta = False
+
+
+class _FakeRobot:
+    def __init__(self) -> None:
+        self.controller = _FakeController()
+
+
+class _FakeNativeEnv:
+    action_spec = ([-1.0] * 7, [1.0] * 7)
+
+    def __init__(self, **kwargs: object) -> None:
+        self.kwargs = kwargs
+        self.robots = [_FakeRobot()]
+        self.actions: list[list[float]] = []
+
+    def reset(self) -> dict[str, object]:
+        self.robots[0].controller.use_delta = False
+        return {"reset": True}
+
+    def set_init_state(self, state: object) -> dict[str, object]:
+        return {"state": state}
+
+    def step(
+        self, action: Sequence[float]
+    ) -> tuple[dict[str, object], float, bool, dict[str, object]]:
+        values = [float(item) for item in action]
+        self.actions.append(values)
+        return {"noop_count": len(self.actions)}, 0.0, False, {}
+
+
 def test_libero_pro_profile_maps_actions_states_score_and_payloads(tmp_path: Path) -> None:
     state = LiberoProRuntimeState(_config(tmp_path), backend=_FakeLiberoBackend())
 
@@ -124,6 +184,141 @@ def test_libero_pro_rejects_incompatible_action_dimension(tmp_path: Path) -> Non
         LiberoProRuntimeState(_config(tmp_path), backend=_BadActionBackend())
 
 
+def test_libero_pro_native_mode_description_advertises_runtime_action_surface(
+    tmp_path: Path,
+) -> None:
+    state = LiberoProRuntimeState(
+        _config(tmp_path, action_mode="native"), backend=_FakeNativeBackend()
+    )
+
+    description = state.describe()
+
+    assert "runtime-action" in description.capabilities
+    assert description.metadata["action_mode"] == "native"
+    assert description.metadata["native_action_space_id"] == NATIVE_ACTION_SPACE_ID
+    assert description.metadata["action_shape"] == [7]
+    assert description.metadata["action_low"] == [-1.0] * 7
+    assert description.metadata["action_high"] == [1.0] * 7
+    assert description.metadata["controller"] == "JOINT_POSITION"
+    assert description.metadata["effective_controller"] == "OSC_POSE"
+    assert description.metadata["task_metadata"] == {
+        "benchmark_name": "libero_pro",
+        "task_order_index": 0,
+        "task_index": 0,
+        "task_name": "pick_up_the_black_bowl",
+        "init_state_index": 2,
+    }
+    assert description.metadata["language"] == "pick up the black bowl"
+    assert description.metadata["horizon"] == 1000
+    assert description.metadata["effective_horizon"] == 1010
+    assert description.metadata["reset_settle_steps"] == 10
+    assert description.metadata["camera_config"] == {
+        "names": ["agentview"],
+        "height": 128,
+        "width": 128,
+    }
+
+
+def test_libero_pro_native_mode_rejects_bad_action_spec(tmp_path: Path) -> None:
+    with pytest.raises(RuntimeError, match="bounds compatible"):
+        LiberoProRuntimeState(
+            _config(tmp_path, action_mode="native"), backend=_BadNativeBoundsBackend()
+        )
+
+
+def test_libero_pro_native_mode_steps_runtime_action_directly(tmp_path: Path) -> None:
+    backend = _FakeNativeBackend()
+    state = LiberoProRuntimeState(_config(tmp_path, action_mode="native"), backend=backend)
+
+    response = state.step(
+        StepRequest(
+            episode_id="episode",
+            tick_id=1,
+            action=RuntimeActionFrame(
+                frame_type="runtime_action",
+                space_id=NATIVE_ACTION_SPACE_ID,
+                values=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7],
+                tick_id=1,
+            ),
+        )
+    )
+
+    assert backend.last_action == [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]
+    assert response.reward == 0.5
+
+
+def test_real_backend_native_reset_runs_lerobot_noops_and_use_delta(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    envs: list[_FakeNativeEnv] = []
+
+    class FakeBenchmark:
+        def get_task(self, task_index: int) -> object:
+            return type("Task", (), {"name": "task", "language": "language"})()
+
+    def fake_env_cls(**kwargs: object) -> _FakeNativeEnv:
+        env = _FakeNativeEnv(**kwargs)
+        envs.append(env)
+        return env
+
+    monkeypatch.setattr(
+        "dimos_libero_pro_sidecar.server.require_libero",
+        lambda *, visualize=False: (
+            type(
+                "BenchmarkModule",
+                (),
+                {"get_benchmark": lambda self, name: lambda order: FakeBenchmark()},
+            )(),
+            fake_env_cls,
+        ),
+    )
+    monkeypatch.setattr("dimos_libero_pro_sidecar.server.ensure_libero_config", lambda *_: None)
+    monkeypatch.setattr("dimos_libero_pro_sidecar.server._load_init_states", lambda *_: ["init"])
+
+    backend = RealLiberoBackend(_config(tmp_path, action_mode="native"))
+    obs = backend.reset(0)
+
+    env = envs[0]
+    assert env.kwargs["controller"] == "OSC_POSE"
+    assert env.kwargs["horizon"] == 1010
+    assert env.robots[0].controller.use_delta is True
+    assert env.actions == [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0]] * 10
+    assert obs == {"noop_count": 10}
+
+
+def test_libero_pro_native_mode_rejects_motor_frame(tmp_path: Path) -> None:
+    state = LiberoProRuntimeState(
+        _config(tmp_path, action_mode="native"), backend=_FakeNativeBackend()
+    )
+
+    with pytest.raises(ValueError, match="native action mode requires RuntimeActionFrame"):
+        state.step(
+            StepRequest(
+                episode_id="episode",
+                tick_id=1,
+                action=MotorActionFrame(robot_id="panda", names=state.motor_names, q=[0.1] * 8),
+            )
+        )
+
+
+def test_libero_pro_motor_mode_rejects_runtime_frame(tmp_path: Path) -> None:
+    state = LiberoProRuntimeState(_config(tmp_path), backend=_FakeLiberoBackend())
+
+    with pytest.raises(ValueError, match="motor action mode requires MotorActionFrame"):
+        state.step(
+            StepRequest(
+                episode_id="episode",
+                tick_id=1,
+                action=RuntimeActionFrame(
+                    frame_type="runtime_action",
+                    space_id=NATIVE_ACTION_SPACE_ID,
+                    values=[0.1] * 7,
+                    tick_id=1,
+                ),
+            )
+        )
+
+
 def test_libero_pro_rejects_unsupported_controller(tmp_path: Path) -> None:
     with pytest.raises(RuntimeError, match="unsupported LIBERO-PRO controller"):
         LiberoProRuntimeState(
@@ -160,6 +355,21 @@ def test_libero_pro_asset_validation_does_not_bootstrap_by_default(tmp_path: Pat
         validate_assets(config)
 
 
+def test_libero_config_is_created_noninteractively(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    config_root = tmp_path / "libero-config"
+    bddl_root = tmp_path / "libero" / "bddl_files"
+    init_states_root = tmp_path / "libero" / "init_files"
+    monkeypatch.setenv("LIBERO_CONFIG_PATH", str(config_root))
+
+    ensure_libero_config(bddl_root, init_states_root)
+
+    config_text = (config_root / "config.yaml").read_text()
+    assert f"bddl_files: {bddl_root}" in config_text
+    assert f"init_states: {init_states_root}" in config_text
+
+
 def test_libero_pro_http_endpoints_with_stubbed_backend(tmp_path: Path) -> None:
     config = _config(tmp_path)
     state = LiberoProRuntimeState(config, backend=_FakeLiberoBackend())
@@ -204,7 +414,11 @@ def test_libero_pro_http_endpoints_with_stubbed_backend(tmp_path: Path) -> None:
 
 
 def _config(
-    tmp_path: Path, *, controller: str = "JOINT_POSITION", visualize: bool = False
+    tmp_path: Path,
+    *,
+    action_mode: str = "motor",
+    controller: str = "JOINT_POSITION",
+    visualize: bool = False,
 ) -> LiberoProRuntimeConfig:
     bddl_root = tmp_path / "bddl"
     init_states_root = tmp_path / "init_states"
@@ -218,6 +432,7 @@ def _config(
         benchmark_name="libero_pro",
         bddl_root=bddl_root,
         init_states_root=init_states_root,
+        action_mode=cast("Literal['motor', 'native']", action_mode),
         controller=controller,
         camera_names=("agentview",),
         init_state_index=2,