Add vision evaluation CI for VLM recipes with PyTorch comparison and perf benchmarks

.github/scripts/generate_matrix.py

@@ -7,9 +7,15 @@
 Script to scan the input directory for files with name "olive_ci.json"
 and generate output that can be set as strategy matrix for github job.
 
+When --changed-files is provided (one file path per line), only recipes
+whose directory contains at least one changed file are included.
+This avoids running all recipes when a PR only touches one recipe folder.
+
 Example:
     python generate_matrix.py <input directory> <ubuntu|windows> <cpu|cuda>
+    python generate_matrix.py <input directory> <ubuntu|windows> <cpu|cuda> --changed-files changed.txt
 """
+import argparse
 import json
 import sys
 from pathlib import Path
@@ -18,18 +24,65 @@
     "requirements_file": "",
 }
 
-dirpath = Path(sys.argv[1])
-os = sys.argv[2]
-device = sys.argv[3]
+parser = argparse.ArgumentParser()
+parser.add_argument("dirpath", type=Path)
+parser.add_argument("os", choices=["ubuntu", "windows"])
+parser.add_argument("device", choices=["cpu", "cuda"])
+parser.add_argument("--changed-files", type=Path, default=None,
+                    help="File containing list of changed file paths (one per line)")
+parser.add_argument("--recipe-filter", type=str, default=None,
+                    help="Only include recipes whose directory name contains this substring")
+args = parser.parse_args()
+
+dirpath = args.dirpath
+os = args.os
+device = args.device
+
+# If changed-files is provided, compute the set of recipe directories that
+# contain at least one changed file. Shared files (e.g. .github/scripts/*)
+# trigger all recipes.
+changed_recipe_dirs = None
+if args.changed_files and args.changed_files.exists():
+    changed_paths = [Path(line.strip()) for line in args.changed_files.read_text().splitlines() if line.strip()]
+    changed_recipe_dirs = set()
+    run_all = False
+    for p in changed_paths:
+        # Changes to shared CI scripts or workflows trigger all recipes
+        if str(p).startswith(".github/"):
+            run_all = True
+            break
+    if run_all:
+        changed_recipe_dirs = None  # None means "run all"
+    else:
+        for filepath in dirpath.rglob("olive_ci.json"):
+            recipe_dir = filepath.parent.relative_to(dirpath)
+            for p in changed_paths:
+                try:
+                    # Check if the changed file is inside this recipe's directory
+                    Path(p).relative_to(recipe_dir)
+                    changed_recipe_dirs.add(str(recipe_dir))
+                    break
+                except ValueError:
+                    continue
 
 recipes = []
 for filepath in dirpath.rglob("olive_ci.json"):
+    recipe_dir = str(filepath.parent.relative_to(dirpath))
+
+    # Skip recipes that weren't touched in this PR
+    if changed_recipe_dirs is not None and recipe_dir not in changed_recipe_dirs:
+        continue
+
+    # Skip recipes that don't match the filter
+    if args.recipe_filter and args.recipe_filter not in recipe_dir:
+        continue
+
     with filepath.open() as strm:
         for config in json.load(strm):
             if config["os"] == os and config["device"] == device:
                 config["name"] = f"{filepath.parent.name} | {config['name']} | {os} | {device}"
                 config["path"] = str(filepath)
-                config["cwd"] = str(filepath.parent.relative_to(dirpath))
+                config["cwd"] = recipe_dir
 
                 for key, value in _defaults.items():
                     if key not in config:

.github/scripts/run_vision_eval.py

@@ -0,0 +1,172 @@
+"""Run MMMU vision evaluation on a pre-built VLM ONNX model.
+
+Uses Olive's genai vision inference path to evaluate multimodal models
+on the MMMU benchmark (Massive Multi-discipline Multimodal Understanding).
+
+Usage:
+    # Eval a pre-built model
+    python run_vision_eval.py --model-path /path/to/model --limit 100
+
+    # Build from olive config + eval
+    python run_vision_eval.py --config cpu/int4/config.json --limit 100
+
+    # GPU eval
+    python run_vision_eval.py --model-path /path/to/model --device gpu --limit 200
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+
+def build_model(config_path: str) -> str:
+    """Build ONNX model via olive run in a subprocess and return model directory path."""
+    print(f"Building model from {config_path}...")
+    start = time.time()
+    result = subprocess.run(
+        [sys.executable, "-m", "olive", "run", "--config", config_path],
+        capture_output=True,
+        text=True,
+    )
+    elapsed = time.time() - start
+
+    if result.returncode != 0:
+        print(f"ERROR: Model build failed (exit code {result.returncode})", file=sys.stderr)
+        print(result.stderr, file=sys.stderr)
+        sys.exit(1)
+
+    config_data = json.loads(Path(config_path).read_text())
+    output_dir = Path(config_data.get("output_dir", "models/output"))
+
+    for p in output_dir.rglob("genai_config.json"):
+        model_dir = str(p.parent)
+        print(f"Model built in {elapsed:.1f}s: {model_dir}")
+        return model_dir
+
+    print(f"ERROR: No genai_config.json found in {output_dir}", file=sys.stderr)
+    sys.exit(1)
+
+
+def run_mmmu_eval(model_path: str, device: str, limit: int | None, subject: str = "Accounting") -> float:
+    """Run MMMU evaluation using Olive's vision evaluator and return accuracy."""
+    from olive.data.config import DataConfig
+    from olive.evaluator.metric import Metric, MetricType
+    from olive.evaluator.olive_evaluator import OnnxEvaluator
+    from olive.hardware import Device
+    from olive.model import ONNXModelHandler
+
+    model_dir = Path(model_path)
+    text_onnx = model_dir / "text.onnx"
+    if not text_onnx.exists():
+        onnx_files = list(model_dir.glob("*.onnx"))
+        if not onnx_files:
+            raise FileNotFoundError(f"No .onnx files in {model_dir}")
+        text_onnx = onnx_files[0]
+
+    model = ONNXModelHandler(model_path=str(text_onnx))
+
+    pre_process_params = {
+        "type": "vision_vqa_pre_process",
+        "image_col": "image_1",
+        "question_col": "question",
+        "answer_col": "answer",
+        "options_col": "options",
+        "system_prompt": "Answer with only the option letter (A, B, C, D, etc.).",
+    }
+    if limit:
+        pre_process_params["limit"] = limit
+
+    data_config = DataConfig(
+        name="mmmu_eval_data",
+        type="HuggingfaceContainer",
+        load_dataset_config={
+            "data_name": "MMMU/MMMU",
+            "subset": subject,
+            "split": "validation",
+        },
+        pre_process_data_config=pre_process_params,
+        dataloader_config={
+            "type": "vision_vqa_dataloader",
+            "batch_size": 1,
+        },
+    )
+    dc = data_config.to_data_container()
+    dataloader = dc.create_dataloader()
+
+    metric = Metric(
+        name="mmmu_accuracy",
+        type=MetricType.ACCURACY,
+        sub_types=[{"name": "exact_match", "priority": 1}],
+        data_config=data_config,
+    )
+
+    eval_device = Device.GPU if device == "gpu" else Device.CPU
+    evaluator = OnnxEvaluator()
+    result = evaluator._evaluate_onnx_accuracy(model, metric, dataloader, device=eval_device)
+
+    # Extract accuracy from result
+    # MetricResult is a dict-based model; find the exact_match key
+    for key, sub_result in result.root.items():
+        if "exact_match" in key:
+            return sub_result.value
+    raise ValueError(f"No exact_match result found. Keys: {list(result.root.keys())}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="MMMU vision evaluation for VLM ONNX models")
+    parser.add_argument("--config", default=None, help="Olive config to build model (skipped if --model-path set)")
+    parser.add_argument("--model-path", default=None, help="Pre-built model directory")
+    parser.add_argument("--device", choices=["cpu", "gpu"], default="cpu")
+    parser.add_argument("--limit", type=int, default=50, help="Samples to evaluate (0=full)")
+    parser.add_argument("--threshold", type=float, default=0.0, help="Min accuracy (fails if below)")
+    parser.add_argument("--subject", default="Accounting", help="MMMU subject subset (default: Accounting)")
+    args = parser.parse_args()
+
+    # Resolve model path
+    if args.model_path:
+        model_path = args.model_path
+    elif args.config:
+        model_path = build_model(args.config)
+    else:
+        print("ERROR: Provide --config or --model-path", file=sys.stderr)
+        sys.exit(1)
+
+    # Verify it's a vision model
+    genai_config = Path(model_path) / "genai_config.json"
+    if not genai_config.exists():
+        print(f"ERROR: genai_config.json not found in {model_path}", file=sys.stderr)
+        sys.exit(1)
+
+    cfg = json.loads(genai_config.read_text())
+    if "vision" not in cfg.get("model", {}):
+        print("WARNING: Model may not be a VLM (no 'vision' field in genai_config.json)")
+
+    limit = args.limit if args.limit > 0 else None
+
+    print(f"\n{'='*60}")
+    print(f"MMMU Evaluation ({args.subject})")
+    print(f"  Model: {model_path}")
+    print(f"  Device: {args.device}")
+    print(f"  Limit: {limit or 'full'}")
+    print(f"{'='*60}")
+
+    start = time.time()
+    acc = run_mmmu_eval(model_path, args.device, limit, args.subject)
+    elapsed = time.time() - start
+
+    status = "PASS" if acc >= args.threshold else "FAIL"
+    print(f"\n  {status}: MMMU ({args.subject}) exact_match = {acc:.4f} ({elapsed:.1f}s)")
+    if args.threshold > 0:
+        print(f"  Threshold: {args.threshold:.4f}")
+
+    if acc < args.threshold:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/main.yml

@@ -15,9 +15,15 @@ on:
       - main
     paths:
       - '**/olive_ci.json'  # Run this workflow only when Olive CI files are modified
+      - '**/config.json'    # Run when recipe configs change
+      - '.github/scripts/**'  # Run when any CI script changes
+      - '.github/workflows/main.yml'  # Run when this workflow changes
   pull_request:
     paths:
       - '**/olive_ci.json'  # Run this workflow only when Olive CI files are modified
+      - '**/config.json'    # Run when recipe configs change
+      - '.github/scripts/**'  # Run when any CI script changes
+      - '.github/workflows/main.yml'  # Run when this workflow changes
 
 env:
   PYTHON_VERSION: "3.10"
@@ -36,34 +42,66 @@ jobs:
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Setup python
       uses: actions/setup-python@v5
       with:
         python-version: ${{ env.PYTHON_VERSION }}
 
+    - name: Get changed files
+      id: changed-files
+      run: |
+        if [ "${{ github.event_name }}" = "pull_request" ]; then
+          git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} > /tmp/changed_files.txt
+        elif [ "${{ github.event_name }}" = "push" ]; then
+          git diff --name-only ${{ github.event.before }} ${{ github.sha }} > /tmp/changed_files.txt
+        else
+          # workflow_dispatch: run all recipes
+          echo "" > /tmp/changed_files.txt
+        fi
+        echo "Changed files:"
+        cat /tmp/changed_files.txt
+
     - name: Scan & generate matrix (Ubuntu + CPU)
       id: ubuntu-gen-cpu-matrix
       run: |
-        matrix=$(python .github/scripts/generate_matrix.py . ubuntu cpu)
+        CHANGED_ARG=""
+        if [ -s /tmp/changed_files.txt ]; then
+          CHANGED_ARG="--changed-files /tmp/changed_files.txt"
+        fi
+        matrix=$(python .github/scripts/generate_matrix.py . ubuntu cpu $CHANGED_ARG --recipe-filter Qwen-Qwen3.5-0.8B)
         echo "ubuntu_cpu_matrix=$matrix" >> $GITHUB_OUTPUT
 
     - name: Scan & generate matrix (Ubuntu + CUDA)
       id: ubuntu-gen-cuda-matrix
       run: |
-        matrix=$(python .github/scripts/generate_matrix.py . ubuntu cuda)
+        CHANGED_ARG=""
+        if [ -s /tmp/changed_files.txt ]; then
+          CHANGED_ARG="--changed-files /tmp/changed_files.txt"
+        fi
+        matrix=$(python .github/scripts/generate_matrix.py . ubuntu cuda $CHANGED_ARG --recipe-filter __disabled__)
         echo "ubuntu_cuda_matrix=$matrix" >> $GITHUB_OUTPUT
 
     - name: Scan & generate matrix (Windows + CPU)
       id: windows-gen-cpu-matrix
       run: |
-        matrix=$(python .github/scripts/generate_matrix.py . windows cpu)
+        CHANGED_ARG=""
+        if [ -s /tmp/changed_files.txt ]; then
+          CHANGED_ARG="--changed-files /tmp/changed_files.txt"
+        fi
+        matrix=$(python .github/scripts/generate_matrix.py . windows cpu $CHANGED_ARG --recipe-filter __disabled__)
         echo "windows_cpu_matrix=$matrix" >> $GITHUB_OUTPUT
 
     - name: Scan & generate matrix (Windows + CUDA)
       id: windows-gen-cuda-matrix
       run: |
-        matrix=$(python .github/scripts/generate_matrix.py . windows cuda)
+        CHANGED_ARG=""
+        if [ -s /tmp/changed_files.txt ]; then
+          CHANGED_ARG="--changed-files /tmp/changed_files.txt"
+        fi
+        matrix=$(python .github/scripts/generate_matrix.py . windows cuda $CHANGED_ARG --recipe-filter __disabled__)
         echo "windows_cuda_matrix=$matrix" >> $GITHUB_OUTPUT
 
   ubuntu-cpu-recipes:
@@ -82,7 +120,7 @@ jobs:
       uses: ./.github/actions/cpu-setup
       with:
         shell: bash
-        python_version: ${{ env.PYTHON_VERSION }}
+        python_version: ${{ matrix.python_version || env.PYTHON_VERSION }}
         hf_cache_key: ${{ runner.os }}-${{ matrix.name }}-hf-cache
         requirements_file: ${{ matrix.cwd }}/${{ matrix.requirements_file }}
 
@@ -106,7 +144,7 @@ jobs:
       uses: ./.github/actions/cuda-setup
       with:
         shell: bash
-        python_version: ${{ env.PYTHON_VERSION }}
+        python_version: ${{ matrix.python_version || env.PYTHON_VERSION }}
         hf_cache_key: ${{ runner.os }}-${{ matrix.name }}-hf-cache
         requirements_file: ${{ matrix.cwd }}/${{ matrix.requirements_file }}
 
@@ -130,7 +168,7 @@ jobs:
       uses: ./.github/actions/cpu-setup
       with:
         shell: cmd
-        python_version: ${{ env.PYTHON_VERSION }}
+        python_version: ${{ matrix.python_version || env.PYTHON_VERSION }}
         hf_cache_key: ${{ runner.os }}-${{ matrix.name }}-hf-cache
         requirements_file: ${{ matrix.cwd }}/${{ matrix.requirements_file }}