Build Copilot SDK

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "typing-extensions>=4.0",
     "pyyaml>=6.0",
     "pydantic>=2.0",
+    "github-copilot-sdk>=0.1.18",
 ]
 
 [project.scripts]

src/bcbench/agent/copilot/agent.py

@@ -1,13 +1,19 @@
 """GitHub Copilot CLI Agent implementation."""
 
+import asyncio
+import json
+import random
 import shutil
 import subprocess
 import sys
 from pathlib import Path
+from typing import cast
 
 import yaml
+from copilot import CopilotClient, MCPServerConfig
+from copilot.generated.session_events import SessionEventType
 
 from bcbench.agent.copilot.metrics import parse_metrics
 from bcbench.agent.shared import build_mcp_config, build_prompt
 from bcbench.config import get_config
 from bcbench.dataset import DatasetEntry
@@ -65,29 +71,68 @@
 
         logger.debug(f"Copilot command args: {cmd_args}")
 
-        result = subprocess.run(
-            cmd_args,
-            cwd=str(repo_path),
-            stderr=subprocess.PIPE,  # only capture stderr where metrics are printed
-            timeout=_config.timeout.agent_execution,
-            check=True,
-        )
+        # Copilot SDK
+        async def run_copilot():
+            client = CopilotClient({"cli_path": copilot_cmd})
+            await client.start()
 
-        if result.stderr:
-            sys.stdout.buffer.write(result.stderr)
-            sys.stdout.buffer.flush()
-        logger.info(f"Copilot CLI run complete for: {entry.instance_id}")
+            if mcp_config_json:
+                raw = json.loads(mcp_config_json)
+                raw_servers = raw.get("mcpServers", {})
+            else:
+                raw_servers = {}
 
-        stderr = result.stderr.decode("utf-8", errors="replace") if result.stderr else ""
-        stderr_lines = stderr.splitlines()
+            mcp_servers = cast(dict[str, MCPServerConfig], raw_servers or {})
 
-        # Find the most recent session log for tool usage parsing
-        session_logs = list(output_dir.glob("process-*.log"))
-        session_log_path = max(session_logs, key=lambda p: p.stat().st_mtime) if session_logs else None
+            session = await client.create_session(
+                {
+                    "model": model,
+                    "mcp_servers": mcp_servers,
+                    "streaming": True,
+                }
+            )
 
-        metrics = parse_metrics(stderr_lines, session_log_path=session_log_path)
+            # Listen for response chunks
+            def handle_event(event):
+                if event.type == SessionEventType.ASSISTANT_MESSAGE_DELTA:
+                    sys.stdout.write(event.data.delta_content)
+                    sys.stdout.flush()
 
-        return metrics, config
+            session.on(handle_event)
+
+            response = await session.send_and_wait(
+                {"prompt": prompt},
+                timeout=_config.timeout.agent_execution,
+            )
+            print()  # newline after streaming
+
+            await client.stop()
+
+        asyncio.run(run_copilot())
+
+        # result = subprocess.run(
+        #     cmd_args,
+        #     cwd=str(repo_path),
+        #     stderr=subprocess.PIPE,  # only capture stderr where metrics are printed
+        #     timeout=_config.timeout.agent_execution,
+        #     check=True,
+        # )
+
+        # if result.stderr:
+        #     sys.stdout.buffer.write(result.stderr)
+        #     sys.stdout.buffer.flush()
+        # logger.info(f"Copilot CLI run complete for: {entry.instance_id}")
+
+        # stderr = result.stderr.decode("utf-8", errors="replace") if result.stderr else ""
+        # stderr_lines = stderr.splitlines()
+
+        # # Find the most recent session log for tool usage parsing
+        # session_logs = list(output_dir.glob("process-*.log"))
+        # session_log_path = max(session_logs, key=lambda p: p.stat().st_mtime) if session_logs else None
+
+        # metrics = parse_metrics(stderr_lines, session_log_path=session_log_path)
+
+        return None, config
     except subprocess.TimeoutExpired:
         logger.error(f"Copilot CLI timed out after {_config.timeout.agent_execution} seconds")
         metrics = AgentMetrics(execution_time=_config.timeout.agent_execution)

src/bcbench/agent/shared/mcp.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 from typing import Any
 
+from copilot import MCPLocalServerConfig, MCPRemoteServerConfig, MCPServerConfig
 from jinja2 import Template
 
 from bcbench.dataset import DatasetEntry
@@ -39,27 +40,27 @@ def cleanup(self) -> None:
 _mcp_server_manager = _ALMcpServerManager()
 
 
-def _build_server_entry(server: dict[str, Any], template_context: dict[str, Any]) -> tuple[str, dict[str, Any]]:
+def _build_server_entry(server: dict[str, Any], template_context: dict[str, Any]) -> tuple[str, dict[str, MCPServerConfig]]:
     server_type: str = server["type"]
     server_name: str = server["name"]
     tools: list[str] = server["tools"]
 
     match server_type:
         case "http":
-            return server_name, {
-                "type": server_type,
-                "url": server["url"],
-                "tools": tools,
-            }
+            return server_name, MCPRemoteServerConfig(
+                tools=tools,
+                url=server["url"],
+                type=server_type
+            )
         case "local":
             args: list[str] = server["args"]
             rendered_args = [Template(arg).render(**template_context) for arg in args]
-            return server_name, {
-                "type": server_type,
-                "command": server["command"],
-                "args": rendered_args,
-                "tools": tools,
-            }
+            return server_name, MCPLocalServerConfig(
+                tools=tools,
+                command=server["command"],
+                args=rendered_args,
+                type=server_type,
+            )
         case _:
             logger.error(f"Unsupported MCP server type: {server_type}, {server}")
             raise AgentError(f"Unsupported MCP server type: {server_type}")