Align identification runner with existing conventions

Two gratuitous divergences from house patterns, surfaced in review:

- Move the hardcoded IDENTIFICATION_TIMEOUT_SECONDS into TimeoutConfig
  (`filepath_identification`), where every other timeout lives; the runner now
  reads `_config.timeout.filepath_identification`.
- De-duplicate the triplicated Copilot-CLI lookup into a shared
  `bcbench.copilot_cli.find_copilot`, now used by the identification runner, the
  code-review judge, and the copilot agent (the judge keeps a `_find_copilot`
  alias so existing test patch points still work).

No behavioural change.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: Magnus Hartvig Grønbech

src/bcbench/agent/copilot/agent.py

@@ -1,7 +1,6 @@
 """GitHub Copilot CLI Agent implementation."""
 
 import os
-import shutil
 import subprocess
 import sys
 from pathlib import Path
@@ -11,6 +10,7 @@
 from bcbench.agent.copilot.metrics import parse_metrics
 from bcbench.agent.shared import build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks
 from bcbench.config import get_config
+from bcbench.copilot_cli import find_copilot
 from bcbench.dataset import BaseDatasetEntry
 from bcbench.exceptions import AgentError, AgentTimeoutError
 from bcbench.logger import get_logger
@@ -41,7 +41,7 @@ def run_copilot_agent(
 
     # Prefer copilot.exe over copilot.bat/copilot.cmd shims on Windows: the .bat shim invokes PowerShell,
     # which re-parses arguments and corrupts prompts containing double quotes (e.g. JSON examples).
-    copilot_cmd = shutil.which("copilot.exe") or shutil.which("copilot.cmd") or shutil.which("copilot")
+    copilot_cmd = find_copilot()
     if not copilot_cmd:
         raise AgentError("Copilot CLI not found in PATH. Please ensure it is installed and available.")
 

src/bcbench/config.py

@@ -72,6 +72,7 @@ class TimeoutConfig:
     test_execution: int
     agent_execution: int
     bcal_execution: int
+    filepath_identification: int
 
     @classmethod
     def default(cls) -> TimeoutConfig:
@@ -83,6 +84,9 @@ def default(cls) -> TimeoutConfig:
             agent_execution=60 * 60,  # 60 minutes for coding agent (claude and copilot) execution
             # Total bcal CLI budget per instance.
             bcal_execution=25 * 60,
+            # Context-free file-path identification; kept below the 20-min workflow step timeout
+            # so a hung run records an error result before the CI step is force-killed.
+            filepath_identification=15 * 60,
         )
 
 

src/bcbench/contamination/runner.py

@@ -10,31 +10,25 @@
 
 from __future__ import annotations
 
-import shutil
 import subprocess
 import tempfile
 from pathlib import Path
 
+from bcbench.config import get_config
 from bcbench.contamination.filepath_identification import FilePathIdentificationResult, build_identification_prompt, parse_prediction
+from bcbench.copilot_cli import find_copilot
 from bcbench.dataset import BaseDatasetEntry
 from bcbench.exceptions import AgentError
 from bcbench.logger import get_logger
 
 logger = get_logger(__name__)
+_config = get_config()
 
-__all__ = ["IDENTIFICATION_TIMEOUT_SECONDS", "load_identification_results", "run_filepath_identification", "save_identification_result"]
-
-# Kept below the workflow step timeout so a hung run raises (and records an error
-# result) before the CI step is force-killed and loses its artifact.
-IDENTIFICATION_TIMEOUT_SECONDS = 15 * 60
-
-
-def _find_copilot() -> str | None:
-    return shutil.which("copilot.exe") or shutil.which("copilot.cmd") or shutil.which("copilot")
+__all__ = ["load_identification_results", "run_filepath_identification", "save_identification_result"]
 
 
 def _run_copilot_context_free(prompt: str, work_dir: Path, model: str) -> str:
-    copilot_cmd = _find_copilot()
+    copilot_cmd = find_copilot()
     if not copilot_cmd:
         raise AgentError("Copilot CLI not found in PATH; cannot run file-path identification")
 
@@ -57,7 +51,7 @@ def _run_copilot_context_free(prompt: str, work_dir: Path, model: str) -> str:
         text=True,
         encoding="utf-8",
         errors="replace",
-        timeout=IDENTIFICATION_TIMEOUT_SECONDS,
+        timeout=_config.timeout.filepath_identification,
         check=True,
     )
 

src/bcbench/copilot_cli.py

@@ -0,0 +1,11 @@
+"""Shared helper for locating the GitHub Copilot CLI executable."""
+
+import shutil
+
+__all__ = ["find_copilot"]
+
+
+def find_copilot() -> str | None:
+    # Prefer copilot.exe over copilot.bat/copilot.cmd shims on Windows: the .bat shim invokes
+    # PowerShell, which re-parses arguments and corrupts prompts containing double quotes.
+    return shutil.which("copilot.exe") or shutil.which("copilot.cmd") or shutil.which("copilot")

src/bcbench/evaluate/codereview_judge.py

@@ -7,11 +7,11 @@
 
 import json
 import re
-import shutil
 import subprocess
 from pathlib import Path
 
 from bcbench.config import get_config
+from bcbench.copilot_cli import find_copilot as _find_copilot
 from bcbench.dataset.codereview import ReviewComment
 from bcbench.exceptions import LLMJudgeError
 
@@ -81,10 +81,6 @@ def _parse_judge_results(result_path: Path, num_pairs: int, stdout: str = "") ->
     return [results_by_pair.get(i + 1, False) for i in range(num_pairs)]
 
 
-def _find_copilot() -> str | None:
-    return shutil.which("copilot.exe") or shutil.which("copilot.cmd") or shutil.which("copilot")
-
-
 def _decode_stream(stream: str | bytes | None) -> str:
     if stream is None:
         return ""

tests/test_plugin_operations.py

@@ -213,9 +213,11 @@ def test_setup_missing_required_key_raises_and_cleans_home(tmp_path):
 @patch("bcbench.agent.copilot.agent.build_al_lsp_plugin", return_value=None)
 @patch("bcbench.agent.copilot.agent.build_mcp_config", return_value=(None, None))
 @patch("bcbench.agent.copilot.agent.build_prompt", return_value="do the task")
-@patch("bcbench.agent.copilot.agent.shutil.which", return_value="copilot")
+@patch("bcbench.agent.copilot.agent.find_copilot", return_value="copilot")
 @patch("bcbench.agent.copilot.agent.subprocess.run")
-def test_copilot_runner_records_plugins_and_sets_home(mock_run, mock_which, mock_prompt, mock_mcp, mock_lsp, mock_instr, mock_skills, mock_agent, mock_hooks, mock_pm, mock_tu, mock_setup, tmp_path):
+def test_copilot_runner_records_plugins_and_sets_home(
+    mock_run, mock_find_copilot, mock_prompt, mock_mcp, mock_lsp, mock_instr, mock_skills, mock_agent, mock_hooks, mock_pm, mock_tu, mock_setup, tmp_path
+):
     from bcbench.agent.copilot.agent import run_copilot_agent
     from bcbench.types import EvaluationCategory