Merge branch 'main' into fix/remove-duplicate-setup-database

Author: zjwu0522

pipeline.py

@@ -8,6 +8,7 @@
 """
 
 import argparse
+import sys
 from datetime import datetime
 from pathlib import Path
 from dotenv import load_dotenv
@@ -19,6 +20,27 @@
 from src.model_config import ModelConfig
 
 
+# Suppress httpcore/anyio cleanup exceptions that don't affect functionality.
+# These "Exception ignored" messages are caused by MCP library's streamablehttp_client
+# timing issues during cleanup, but don't impact actual task execution.
+def _suppress_cleanup_exceptions(unraisable):
+    """Suppress known cleanup exceptions from httpcore/anyio."""
+    msg = str(unraisable.exc_value)
+    if any(
+        pattern in msg
+        for pattern in [
+            "async generator ignored GeneratorExit",
+            "cancel scope in a different task",
+            "no running event loop",
+        ]
+    ):
+        return  # Silently ignore
+    # Use default handler for other exceptions
+    sys.__unraisablehook__(unraisable)
+
+
+sys.unraisablehook = _suppress_cleanup_exceptions
+
 # Initialize logger
 logger = get_logger(__name__)
 
@@ -74,7 +96,10 @@ def main():
 
     # Execution configuration
     parser.add_argument(
-        "--timeout", type=int, default=3600, help="Timeout in seconds for agent execution"
+        "--timeout",
+        type=int,
+        default=3600,
+        help="Timeout in seconds for agent execution",
     )
     parser.add_argument(
         "--compaction-token",
@@ -125,7 +150,9 @@ def main():
         )
 
     logger.info("MCPMark Evaluation")
-    logger.info(f"Experiment: {args.exp_name} | {len(model_list)} Model(s): {', '.join(model_list)}")
+    logger.info(
+        f"Experiment: {args.exp_name} | {len(model_list)} Model(s): {', '.join(model_list)}"
+    )
     logger.info(f"Task suite: {args.task_suite}")
     if args.k > 1:
         logger.info(f"Running {args.k} evaluation runs for pass@k metrics")
@@ -136,7 +163,7 @@ def main():
             logger.info(f"\n{'=' * 80}")
             logger.info(f"Starting Run {run_idx}/{args.k}")
             logger.info(f"{'=' * 80}\n")
-            
+
             # For k-runs, results/{exp}/{mcp}__{model}/run-N
             run_exp_name = f"run-{run_idx}"
             run_output_dir = args.output_dir / args.exp_name
@@ -149,7 +176,9 @@ def main():
         for i, model in enumerate(model_list, 1):
             logger.info(f"\n{'=' * 60}")
             if args.k > 1:
-                logger.info(f"Run {run_idx}/{args.k} | Model {i}/{len(model_list)}: {model}")
+                logger.info(
+                    f"Run {run_idx}/{args.k} | Model {i}/{len(model_list)}: {model}"
+                )
             else:
                 logger.info(f"Starting evaluation {i}/{len(model_list)}: {model}")
             logger.info(f"{'=' * 60}\n")
@@ -168,14 +197,14 @@ def main():
             )
 
             pipeline.run_evaluation(args.tasks)
-            logger.info(
-                f"📁 Results: {pipeline.base_experiment_dir}"
-            )
+            logger.info(f"📁 Results: {pipeline.base_experiment_dir}")
 
     logger.info(f"\n{'=' * 60}")
     if args.k > 1:
         logger.info(f"✓ All {args.k} runs completed for {len(model_list)} model(s)")
-        logger.info(f"Run `python -m src.aggregators.aggregate_results --exp-name {args.exp_name}` to compute all metrics")
+        logger.info(
+            f"Run `python -m src.aggregators.aggregate_results --exp-name {args.exp_name}` to compute all metrics"
+        )
     else:
         logger.info(f"✓ All evaluations completed for {len(model_list)} model(s)")
     logger.info(f"{'=' * 60}")

pyproject.toml

@@ -4,7 +4,7 @@ name = "MCPMark"
 requires-python = ">= 3.11"
 version = "0.0.1"
 dependencies = [
-  "notion-client>=2.4.0",
+  "notion-client==2.4.0",
   "playwright>=1.43.0",
   "seaborn>=0.12.0",
   "matplotlib>=3.7.0",

requirements.txt

@@ -19,7 +19,14 @@
 class BaseMCPAgent(ABC):
     """Base class with shared functionality for MCPMark agents."""
 
-    STDIO_SERVICES = ["notion", "filesystem", "playwright", "playwright_webarena", "postgres", "insforge"]
+    STDIO_SERVICES = [
+        "notion",
+        "filesystem",
+        "playwright",
+        "playwright_webarena",
+        "postgres",
+        "insforge",
+    ]
     HTTP_SERVICES = ["github", "supabase"]
     DEFAULT_TIMEOUT = 600
     COMPACTION_DISABLED_TOKEN = 999_999_999
@@ -70,7 +77,11 @@ def __init__(
         )
 
         # Warn if Gemini 3 model uses unsupported reasoning_effort value
-        if self._is_gemini_3_model() and self.reasoning_effort not in ["default", "low", "high"]:
+        if self._is_gemini_3_model() and self.reasoning_effort not in [
+            "default",
+            "low",
+            "high",
+        ]:
             logger.warning(
                 "Gemini 3 models only support reasoning_effort 'low' or 'high', "
                 "got '%s'. LiteLLM may map this to the nearest supported value.",
@@ -179,7 +190,11 @@ def _create_stdio_server(self) -> MCPStdioServer:
                 raise ValueError("Test directory required for filesystem service")
             return MCPStdioServer(
                 command="npx",
-                args=["-y", "@modelcontextprotocol/server-filesystem", str(test_directory)],
+                args=[
+                    "-y",
+                    "@modelcontextprotocol/server-filesystem",
+                    str(test_directory),
+                ],
             )
 
         if self.mcp_service in ("playwright", "playwright_webarena"):
@@ -208,10 +223,14 @@ def _create_stdio_server(self) -> MCPStdioServer:
             port = self.service_config.get("port", 5432)
             username = self.service_config.get("username")
             password = self.service_config.get("password")
-            database = self.service_config.get("current_database") or self.service_config.get("database")
+            database = self.service_config.get(
+                "current_database"
+            ) or self.service_config.get("database")
             if not all([username, password, database]):
                 raise ValueError("PostgreSQL requires username, password, and database")
-            database_url = f"postgresql://{username}:{password}@{host}:{port}/{database}"
+            database_url = (
+                f"postgresql://{username}:{password}@{host}:{port}/{database}"
+            )
             return MCPStdioServer(
                 command="pipx",
                 args=["run", "postgres-mcp", "--access-mode=unrestricted"],
@@ -259,11 +278,16 @@ def _count_prompt_tokens_litellm(self, messages: List[Dict[str, Any]]) -> int:
         try:
             from litellm import token_counter
 
-            return int(token_counter(model=self.litellm_input_model_name, messages=messages) or 0)
+            return int(
+                token_counter(model=self.litellm_input_model_name, messages=messages)
+                or 0
+            )
         except Exception:  # pragma: no cover - best effort
             return 0
 
-    def _convert_to_sdk_format(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    def _convert_to_sdk_format(
+        self, messages: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         sdk_format: List[Dict[str, Any]] = []
         function_call_map: Dict[str, str] = {}
 
@@ -305,7 +329,9 @@ def _convert_to_sdk_format(self, messages: List[Dict[str, Any]]) -> List[Dict[st
                         for item in user_content:
                             if isinstance(item, dict) and item.get("type") == "text":
                                 text_parts.append(item.get("text", ""))
-                        sdk_format.append({"content": "\n".join(text_parts), "role": "user"})
+                        sdk_format.append(
+                            {"content": "\n".join(text_parts), "role": "user"}
+                        )
                 else:
                     sdk_format.append({"content": user_content, "role": "user"})
 
@@ -324,7 +350,9 @@ def _convert_to_sdk_format(self, messages: List[Dict[str, Any]]) -> List[Dict[st
                             elif block.get("type") == "thinking":
                                 thinking_text = block.get("thinking", "")
                                 if thinking_text:
-                                    text_parts.append(f"<think>\n{thinking_text}\n</think>")
+                                    text_parts.append(
+                                        f"<think>\n{thinking_text}\n</think>"
+                                    )
                             elif block.get("type") == "tool_use":
                                 claude_tool_uses.append(block)
                     content = "\n".join(text_parts)
@@ -364,7 +392,9 @@ def _convert_to_sdk_format(self, messages: List[Dict[str, Any]]) -> List[Dict[st
                         func_name = tool_call.get("function", {}).get("name", "")
                         sdk_format.append(
                             {
-                                "arguments": tool_call.get("function", {}).get("arguments", "{}"),
+                                "arguments": tool_call.get("function", {}).get(
+                                    "arguments", "{}"
+                                ),
                                 "call_id": call_id,
                                 "name": func_name,
                                 "type": "function_call",
@@ -422,7 +452,9 @@ def _convert_to_sdk_format(self, messages: List[Dict[str, Any]]) -> List[Dict[st
 
         return sdk_format
 
-    def _convert_to_anthropic_format(self, tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    def _convert_to_anthropic_format(
+        self, tools: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         anthropic_tools = []
         for tool in tools:
             anthropic_tool = {
@@ -445,7 +477,9 @@ def _is_gemini_3_model(self) -> bool:
         model_lower = self.litellm_input_model_name.lower()
         return "gemini-3" in model_lower or "gemini/gemini-3" in model_lower
 
-    def _simplify_schema_for_gemini(self, schema: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+    def _simplify_schema_for_gemini(
+        self, schema: Optional[Dict[str, Any]]
+    ) -> Dict[str, Any]:
         if not isinstance(schema, dict):
             return schema or {}
 
@@ -464,14 +498,18 @@ def _simplify_schema_for_gemini(self, schema: Optional[Dict[str, Any]]) -> Dict[
                 simplified[key] = self._simplify_schema_for_gemini(value)
             elif isinstance(value, list) and key not in ("required", "enum"):
                 simplified[key] = [
-                    self._simplify_schema_for_gemini(item) if isinstance(item, dict) else item
+                    self._simplify_schema_for_gemini(item)
+                    if isinstance(item, dict)
+                    else item
                     for item in value
                 ]
             else:
                 simplified[key] = value
         return simplified
 
-    def _convert_to_openai_format(self, tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    def _convert_to_openai_format(
+        self, tools: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         functions = []
         is_gemini = self._is_gemini_model()