diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 1f061b90..df05975d 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -5,6 +5,11 @@ "version": "v8", "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd" }, + "github/gh-aw-actions/setup@v0.81.6": { + "repo": "github/gh-aw-actions/setup", + "version": "v0.81.6", + "sha": "ba6380cc6e5be5d21677bebe04d52fb48e3abec7" + }, "githubnext/gh-aw/actions/setup@v0.36.0": { "repo": "githubnext/gh-aw/actions/setup", "version": "v0.36.0", diff --git a/.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py/SKILL.md index 04a975fe..98b28f5c 100644 --- a/.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py/SKILL.md @@ -347,7 +347,7 @@ if __name__ == "__main__": ## Best Practices -1. **This SDK is async-first** — use `async def` handlers and `async with` throughout. +1. **This SDK is async-first — use `async def` handlers and `async with` throughout.** 2. **Always use context managers for clients and async credentials.** Wrap every client in `with Client(...) as client:` (sync) or `async with Client(...) as client:` (async). For async `DefaultAzureCredential` from `azure.identity.aio`, also use `async with credential:` so tokens and transports are cleaned up. ## Reference Files diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/SKILL.md index f48d8a11..9caadcc9 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/SKILL.md @@ -240,3 +240,10 @@ request = AnalyzeTextOptions( 7. **Log analysis results** for audit and improvement 8. **Consider 8-severity mode** for finer-grained control 9. **Pre-moderate AI outputs** before showing to users + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/capabilities.md new file mode 100644 index 00000000..529c1776 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/capabilities.md @@ -0,0 +1,34 @@ +# azure-ai-contentsafety-py capability coverage + +**SDK/package**: `azure-ai-contentsafety` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Analyze Text` +- `Analyze Image` +- `Text Blocklist Management` +- `Severity Levels` + +## Non-hero scenarios + +- `Harm Categories`: | Category | Description | + See: [`non-hero-scenarios.md#harm-categories`](non-hero-scenarios.md#harm-categories) +- `Severity Scale`: | Level | Text Range | Image Range | Meaning | + See: [`non-hero-scenarios.md#severity-scale`](non-hero-scenarios.md#severity-scale) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..3d08cd45 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/non-hero-scenarios.md @@ -0,0 +1,29 @@ +# azure-ai-contentsafety-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Harm Categories + +| Category | Description | +|----------|-------------| +| `Hate` | Attacks based on identity (race, religion, gender, etc.) | +| `Sexual` | Sexual content, relationships, anatomy | +| `Violence` | Physical harm, weapons, injury | +| `SelfHarm` | Self-injury, suicide, eating disorders | + +## Severity Scale + +| Level | Text Range | Image Range | Meaning | +|-------|------------|-------------|---------| +| 0 | Safe | Safe | No harmful content | +| 2 | Low | Low | Mild references | +| 4 | Medium | Medium | Moderate content | +| 6 | High | High | Severe content | + +## Client Types + +| Client | Purpose | +|--------|---------| +| `ContentSafetyClient` | Analyze text and images | +| `BlocklistClient` | Manage custom blocklists | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/SKILL.md index 1998fba3..5b59e23e 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/SKILL.md @@ -292,3 +292,10 @@ from azure.ai.contentunderstanding.models import ( 7. **Use async client** for high-throughput scenarios with `azure.identity.aio` credentials 8. **Handle long-running operations** — video/audio analysis can take minutes 9. **Use URL sources** when possible to avoid upload overhead + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/capabilities.md new file mode 100644 index 00000000..d03bbc13 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/capabilities.md @@ -0,0 +1,46 @@ +# azure-ai-contentunderstanding-py capability coverage + +**SDK/package**: `azure-ai-contentunderstanding` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Core Workflow` +- `Prebuilt Analyzers` +- `Analyze Document` +- `Access Document Content Details` + +## Non-hero scenarios + +- `Analyze Image`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#analyze-image`](non-hero-scenarios.md#analyze-image) +- `Analyze Video`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#analyze-video`](non-hero-scenarios.md#analyze-video) +- `Analyze Audio`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#analyze-audio`](non-hero-scenarios.md#analyze-audio) +- `Custom Analyzers`: Create custom analyzers with field schemas for specialized extraction: + See: [`non-hero-scenarios.md#custom-analyzers`](non-hero-scenarios.md#custom-analyzers) +- `Analyzer Management`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#analyzer-management`](non-hero-scenarios.md#analyzer-management) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Content Types`: | Class | For | Provides | + See: [`non-hero-scenarios.md#content-types`](non-hero-scenarios.md#content-types) +- `Model Imports`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#model-imports`](non-hero-scenarios.md#model-imports) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..f5a6a4fb --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/non-hero-scenarios.md @@ -0,0 +1,175 @@ +# azure-ai-contentunderstanding-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Analyze Image + +```python +from azure.ai.contentunderstanding.models import AnalyzeInput + +poller = client.begin_analyze( + analyzer_id="prebuilt-imageSearch", + inputs=[AnalyzeInput(url="https://example.com/image.jpg")] +) +result = poller.result() +content = result.contents[0] +print(content.markdown) +``` + +## Analyze Video + +```python +from azure.ai.contentunderstanding.models import AnalyzeInput + +poller = client.begin_analyze( + analyzer_id="prebuilt-videoSearch", + inputs=[AnalyzeInput(url="https://example.com/video.mp4")] +) + +result = poller.result() + +# Access video content (AudioVisualContent) +content = result.contents[0] + +# Get transcript phrases with timing +for phrase in content.transcript_phrases: + print(f"[{phrase.start_time} - {phrase.end_time}]: {phrase.text}") + +# Get key frames (for video) +for frame in content.key_frames: + print(f"Frame at {frame.time}: {frame.description}") +``` + +## Analyze Audio + +```python +from azure.ai.contentunderstanding.models import AnalyzeInput + +poller = client.begin_analyze( + analyzer_id="prebuilt-audioSearch", + inputs=[AnalyzeInput(url="https://example.com/audio.mp3")] +) + +result = poller.result() + +# Access audio transcript +content = result.contents[0] +for phrase in content.transcript_phrases: + print(f"[{phrase.start_time}] {phrase.text}") +``` + +## Custom Analyzers + +Create custom analyzers with field schemas for specialized extraction: + +```python +# Create custom analyzer +analyzer = client.create_analyzer( + analyzer_id="my-invoice-analyzer", + analyzer={ + "description": "Custom invoice analyzer", + "base_analyzer_id": "prebuilt-documentSearch", + "field_schema": { + "fields": { + "vendor_name": {"type": "string"}, + "invoice_total": {"type": "number"}, + "line_items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": {"type": "string"}, + "amount": {"type": "number"} + } + } + } + } + } + } +) + +# Use custom analyzer +from azure.ai.contentunderstanding.models import AnalyzeInput + +poller = client.begin_analyze( + analyzer_id="my-invoice-analyzer", + inputs=[AnalyzeInput(url="https://example.com/invoice.pdf")] +) + +result = poller.result() + +# Access extracted fields +print(result.fields["vendor_name"]) +print(result.fields["invoice_total"]) +``` + +## Analyzer Management + +```python +# List all analyzers +analyzers = client.list_analyzers() +for analyzer in analyzers: + print(f"{analyzer.analyzer_id}: {analyzer.description}") + +# Get specific analyzer +analyzer = client.get_analyzer("prebuilt-documentSearch") + +# Delete custom analyzer +client.delete_analyzer("my-custom-analyzer") +``` + +## Async Client + +```python +import asyncio +import os +from azure.ai.contentunderstanding.aio import ContentUnderstandingClient +from azure.ai.contentunderstanding.models import AnalyzeInput +from azure.identity.aio import DefaultAzureCredential + +async def analyze_document(): + endpoint = os.environ["CONTENTUNDERSTANDING_ENDPOINT"] + async with DefaultAzureCredential() as credential: + async with ContentUnderstandingClient( + endpoint=endpoint, + credential=credential + ) as client: + poller = await client.begin_analyze( + analyzer_id="prebuilt-documentSearch", + inputs=[AnalyzeInput(url="https://example.com/doc.pdf")] + ) + result = await poller.result() + content = result.contents[0] + return content.markdown + +asyncio.run(analyze_document()) +``` + +## Content Types + +| Class | For | Provides | +|-------|-----|----------| +| `DocumentContent` | PDF, images, Office docs | Pages, tables, figures, paragraphs | +| `AudioVisualContent` | Audio, video files | Transcript phrases, timing, key frames | + +Both derive from `MediaContent` which provides basic info and markdown representation. + +## Model Imports + +```python +from azure.ai.contentunderstanding.models import ( + AnalyzeInput, + AnalyzeResult, + MediaContentKind, + DocumentContent, + AudioVisualContent, +) +``` + +## Client Types + +| Client | Purpose | +|--------|---------| +| `ContentUnderstandingClient` | Sync client for all operations | +| `ContentUnderstandingClient` (aio) | Async client for all operations | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/SKILL.md index 74a373cb..d875833e 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/SKILL.md @@ -96,4 +96,11 @@ with ConversationAnalysisClient(endpoint, credential) as client: } ) - print(f"Top intent: {result['result']['prediction']['topIntent']}") \ No newline at end of file + print(f"Top intent: {result['result']['prediction']['topIntent']}") + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/capabilities.md new file mode 100644 index 00000000..abaae4b5 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/capabilities.md @@ -0,0 +1,27 @@ +# azure-ai-language-conversations-py capability coverage + +**SDK/package**: `azure-ai-language-conversations` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Core workflow` + +## Non-hero scenarios + +- `Operational hardening`: Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. + See: [`non-hero-scenarios.md#operational-hardening`](non-hero-scenarios.md#operational-hardening) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..d7383045 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/non-hero-scenarios.md @@ -0,0 +1,8 @@ +# azure-ai-language-conversations-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Operational hardening + +Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/SKILL.md index 64ae2381..c3b6d0e1 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/SKILL.md @@ -299,3 +299,10 @@ print(f"Default: {default_ds.name}") 7. **Register models** after successful training jobs 8. **Use pipelines** for multi-step workflows 9. **Tag resources** for organization and cost tracking + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/capabilities.md new file mode 100644 index 00000000..35896a80 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/capabilities.md @@ -0,0 +1,38 @@ +# azure-ai-ml-py capability coverage + +**SDK/package**: `azure-ai-ml` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Workspace Management` +- `Data Assets` +- `Model Registry` +- `Compute` + +## Non-hero scenarios + +- `Jobs`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#jobs`](non-hero-scenarios.md#jobs) +- `Pipelines`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#pipelines`](non-hero-scenarios.md#pipelines) +- `Environments`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#environments`](non-hero-scenarios.md#environments) +- `Datastores`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#datastores`](non-hero-scenarios.md#datastores) +- `MLClient Operations`: | Property | Operations | + See: [`non-hero-scenarios.md#mlclient-operations`](non-hero-scenarios.md#mlclient-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..f363f7b1 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/non-hero-scenarios.md @@ -0,0 +1,104 @@ +# azure-ai-ml-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Jobs + +### Command Job + +```python +from azure.ai.ml import command, Input + +job = command( + code="./src", + command="python train.py --data ${{inputs.data}} --lr ${{inputs.learning_rate}}", + inputs={ + "data": Input(type="uri_folder", path="azureml:my-dataset:1"), + "learning_rate": 0.01 + }, + environment="AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest", + compute="cpu-cluster", + display_name="training-job" +) + +returned_job = ml_client.jobs.create_or_update(job) +print(f"Job URL: {returned_job.studio_url}") +``` + +### Monitor Job + +```python +ml_client.jobs.stream(returned_job.name) +``` + +## Pipelines + +```python +from azure.ai.ml import dsl, Input, Output +from azure.ai.ml.entities import Pipeline + +@dsl.pipeline( + compute="cpu-cluster", + description="Training pipeline" +) +def training_pipeline(data_input): + prep_step = prep_component(data=data_input) + train_step = train_component( + data=prep_step.outputs.output_data, + learning_rate=0.01 + ) + return {"model": train_step.outputs.model} + +pipeline = training_pipeline( + data_input=Input(type="uri_folder", path="azureml:my-dataset:1") +) + +pipeline_job = ml_client.jobs.create_or_update(pipeline) +``` + +## Environments + +### Create Custom Environment + +```python +from azure.ai.ml.entities import Environment + +env = Environment( + name="my-env", + version="1", + image="mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04", + conda_file="./environment.yml" +) + +ml_client.environments.create_or_update(env) +``` + +## Datastores + +### List Datastores + +```python +for ds in ml_client.datastores.list(): + print(f"{ds.name}: {ds.type}") +``` + +### Get Default Datastore + +```python +default_ds = ml_client.datastores.get_default() +print(f"Default: {default_ds.name}") +``` + +## MLClient Operations + +| Property | Operations | +|----------|------------| +| `workspaces` | create, get, list, delete | +| `jobs` | create_or_update, get, list, stream, cancel | +| `models` | create_or_update, get, list, archive | +| `data` | create_or_update, get, list | +| `compute` | begin_create_or_update, get, list, delete | +| `environments` | create_or_update, get, list | +| `datastores` | create_or_update, get, list, get_default | +| `components` | create_or_update, get, list | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/SKILL.md index dc9d0d92..3f7ec0d3 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/SKILL.md @@ -252,3 +252,10 @@ async def analyze(): 5. **Use async client** for high-throughput scenarios 6. **Handle document errors** — results list may contain errors for some docs 7. **Specify language** when known to improve accuracy + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/capabilities.md new file mode 100644 index 00000000..19ce159a --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/capabilities.md @@ -0,0 +1,40 @@ +# azure-ai-textanalytics-py capability coverage + +**SDK/package**: `azure-ai-textanalytics` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Sentiment Analysis` +- `Entity Recognition` +- `PII Detection` +- `Key Phrase Extraction` + +## Non-hero scenarios + +- `Language Detection`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#language-detection`](non-hero-scenarios.md#language-detection) +- `Healthcare Text Analytics`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#healthcare-text-analytics`](non-hero-scenarios.md#healthcare-text-analytics) +- `Multiple Analysis (Batch)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#multiple-analysis-batch`](non-hero-scenarios.md#multiple-analysis-batch) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) +- `Available Operations`: | Method | Description | + See: [`non-hero-scenarios.md#available-operations`](non-hero-scenarios.md#available-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..461cb890 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/non-hero-scenarios.md @@ -0,0 +1,104 @@ +# azure-ai-textanalytics-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Language Detection + +```python +documents = ["Ce document est en francais.", "This is written in English."] + +result = client.detect_language(documents) + +for doc in result: + if not doc.is_error: + print(f"Language: {doc.primary_language.name} ({doc.primary_language.iso6391_name})") + print(f"Confidence: {doc.primary_language.confidence_score:.2f}") +``` + +## Healthcare Text Analytics + +```python +documents = ["Patient has diabetes and was prescribed metformin 500mg twice daily."] + +poller = client.begin_analyze_healthcare_entities(documents) +result = poller.result() + +for doc in result: + if not doc.is_error: + for entity in doc.entities: + print(f"Entity: {entity.text}") + print(f" Category: {entity.category}") + print(f" Normalized: {entity.normalized_text}") + + # Entity links (UMLS, etc.) + for link in entity.data_sources: + print(f" Link: {link.name} - {link.entity_id}") +``` + +## Multiple Analysis (Batch) + +```python +from azure.ai.textanalytics import ( + RecognizeEntitiesAction, + ExtractKeyPhrasesAction, + AnalyzeSentimentAction +) + +documents = ["Microsoft announced new Azure AI features at Build conference."] + +poller = client.begin_analyze_actions( + documents, + actions=[ + RecognizeEntitiesAction(), + ExtractKeyPhrasesAction(), + AnalyzeSentimentAction() + ] +) + +results = poller.result() +for doc_results in results: + for result in doc_results: + if result.kind == "EntityRecognition": + print(f"Entities: {[e.text for e in result.entities]}") + elif result.kind == "KeyPhraseExtraction": + print(f"Key phrases: {result.key_phrases}") + elif result.kind == "SentimentAnalysis": + print(f"Sentiment: {result.sentiment}") +``` + +## Async Client + +```python +from azure.ai.textanalytics.aio import TextAnalyticsClient +from azure.identity.aio import DefaultAzureCredential + +async def analyze(): + async with DefaultAzureCredential() as credential: + async with TextAnalyticsClient( + endpoint=endpoint, + credential=credential + ) as client: + result = await client.analyze_sentiment(documents) + # Process results... +``` + +## Client Types + +| Client | Purpose | +|--------|---------| +| `TextAnalyticsClient` | All text analytics operations | +| `TextAnalyticsClient` (aio) | Async version | + +## Available Operations + +| Method | Description | +|--------|-------------| +| `analyze_sentiment` | Sentiment analysis with opinion mining | +| `recognize_entities` | Named entity recognition | +| `recognize_pii_entities` | PII detection and redaction | +| `recognize_linked_entities` | Entity linking to Wikipedia | +| `extract_key_phrases` | Key phrase extraction | +| `detect_language` | Language detection | +| `begin_analyze_healthcare_entities` | Healthcare NLP (long-running) | +| `begin_analyze_actions` | Multiple analyses in batch | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/SKILL.md index a6fd15e7..6f43ff43 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/SKILL.md @@ -24,12 +24,21 @@ pip install azure-ai-transcription ```bash TRANSCRIPTION_ENDPOINT=https://.cognitiveservices.azure.com -TRANSCRIPTION_KEY= +TRANSCRIPTION_KEY= # Required for all auth paths (this SDK currently uses key auth) ``` -## Authentication +## Authentication & Lifecycle -Use subscription key authentication (DefaultAzureCredential is not supported for this client): +> **🔑 Two rules apply to every code sample below:** +> +> 1. **This SDK currently requires API-key authentication.** `DefaultAzureCredential`/Entra ID isn't available for this client yet, so use `TRANSCRIPTION_KEY` from environment variables (never hardcoded in code). +> 2. **Wrap every client in a context manager** so HTTP transports and sockets are released deterministically: +> - Sync: `with (...) as client:` +> - Async: `async with (...) as client:` +> +> Snippets may abbreviate this setup, but production code should always follow both rules. + +Use subscription key authentication: ```python import os @@ -88,3 +97,10 @@ with TranscriptionClient( 6. **Specify language** to improve recognition accuracy 7. **Handle streaming backpressure** for real-time transcription 8. **Close transcription sessions** when complete + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/capabilities.md new file mode 100644 index 00000000..9f35147f --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/capabilities.md @@ -0,0 +1,28 @@ +# azure-ai-transcription-py capability coverage + +**SDK/package**: `azure-ai-transcription` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Transcription (Batch)` +- `Transcription (Real-time)` + +## Non-hero scenarios + +- `Operational hardening`: Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. + See: [`non-hero-scenarios.md#operational-hardening`](non-hero-scenarios.md#operational-hardening) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..a1532705 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/non-hero-scenarios.md @@ -0,0 +1,8 @@ +# azure-ai-transcription-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Operational hardening + +Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/SKILL.md index 07379f43..4cf2eca9 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/SKILL.md @@ -276,3 +276,10 @@ async def translate_documents(): 7. **Separate target containers** for each language 8. **Use async client** for multiple concurrent jobs 9. **Check supported formats** before submitting documents + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/capabilities.md new file mode 100644 index 00000000..d0ead4f4 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/capabilities.md @@ -0,0 +1,44 @@ +# azure-ai-translation-document-py capability coverage + +**SDK/package**: `azure-ai-translation-document` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Basic Document Translation` +- `Multiple Target Languages` +- `Translate Single Document` +- `Check Translation Status` + +## Non-hero scenarios + +- `List Document Statuses`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-document-statuses`](non-hero-scenarios.md#list-document-statuses) +- `Cancel Translation`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#cancel-translation`](non-hero-scenarios.md#cancel-translation) +- `Using Glossary`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#using-glossary`](non-hero-scenarios.md#using-glossary) +- `Supported Document Formats`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#supported-document-formats`](non-hero-scenarios.md#supported-document-formats) +- `Supported Languages`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#supported-languages`](non-hero-scenarios.md#supported-languages) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Supported Formats`: | Category | Formats | + See: [`non-hero-scenarios.md#supported-formats`](non-hero-scenarios.md#supported-formats) +- `Storage Requirements`: - Source and target containers must be Azure Blob Storage + See: [`non-hero-scenarios.md#storage-requirements`](non-hero-scenarios.md#storage-requirements) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..cf9cb490 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/non-hero-scenarios.md @@ -0,0 +1,105 @@ +# azure-ai-translation-document-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## List Document Statuses + +```python +# Get status of individual documents in a job +operation_id = poller.id +document_statuses = client.list_document_statuses(operation_id) + +for doc in document_statuses: + print(f"Document: {doc.source_document_url}") + print(f" Status: {doc.status}") + print(f" Translated to: {doc.translated_to}") + if doc.error: + print(f" Error: {doc.error.message}") +``` + +## Cancel Translation + +```python +# Cancel a running translation +client.cancel_translation(operation_id) +``` + +## Using Glossary + +```python +from azure.ai.translation.document import TranslationGlossary + +poller = client.begin_translation( + inputs=[ + DocumentTranslationInput( + source_url=source_url, + targets=[ + TranslationTarget( + target_url=target_url, + language="es", + glossaries=[ + TranslationGlossary( + glossary_url="https://.blob.core.windows.net/glossary/terms.csv?", + file_format="csv" + ) + ] + ) + ] + ) + ] +) +``` + +## Supported Document Formats + +```python +# Get supported formats +formats = client.get_supported_document_formats() + +for fmt in formats: + print(f"Format: {fmt.format}") + print(f" Extensions: {fmt.file_extensions}") + print(f" Content types: {fmt.content_types}") +``` + +## Supported Languages + +```python +# Get supported languages +languages = client.get_supported_languages() + +for lang in languages: + print(f"Language: {lang.name} ({lang.code})") +``` + +## Async Client + +```python +from azure.ai.translation.document.aio import DocumentTranslationClient +from azure.identity.aio import DefaultAzureCredential + +async def translate_documents(): + async with DefaultAzureCredential() as credential: + async with DocumentTranslationClient( + endpoint=endpoint, + credential=credential, + ) as client: + poller = await client.begin_translation(inputs=[...]) + result = await poller.result() +``` + +## Supported Formats + +| Category | Formats | +|----------|---------| +| Documents | DOCX, PDF, PPTX, XLSX, HTML, TXT, RTF | +| Structured | CSV, TSV, JSON, XML | +| Localization | XLIFF, XLF, MHTML | + +## Storage Requirements + +- Source and target containers must be Azure Blob Storage +- Use SAS tokens with appropriate permissions: + - Source: Read, List + - Target: Write, List diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/SKILL.md index 11d6f6fc..e8ed670c 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/SKILL.md @@ -298,3 +298,10 @@ async def translate_text(): 7. **Handle profanity** appropriately for your application 8. **Use html text_type** when translating HTML content 9. **Include alignment** for applications needing word mapping + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/capabilities.md new file mode 100644 index 00000000..379de29c --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/capabilities.md @@ -0,0 +1,44 @@ +# azure-ai-translation-text-py capability coverage + +**SDK/package**: `azure-ai-translation-text` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Basic Translation` +- `Translate to Multiple Languages` +- `Specify Source Language` +- `Language Detection` + +## Non-hero scenarios + +- `Transliteration`: Convert text from one script to another: + See: [`non-hero-scenarios.md#transliteration`](non-hero-scenarios.md#transliteration) +- `Dictionary Lookup`: Find alternate translations and definitions: + See: [`non-hero-scenarios.md#dictionary-lookup`](non-hero-scenarios.md#dictionary-lookup) +- `Dictionary Examples`: Get usage examples for translations: + See: [`non-hero-scenarios.md#dictionary-examples`](non-hero-scenarios.md#dictionary-examples) +- `Get Supported Languages`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#get-supported-languages`](non-hero-scenarios.md#get-supported-languages) +- `Break Sentence`: Identify sentence boundaries: + See: [`non-hero-scenarios.md#break-sentence`](non-hero-scenarios.md#break-sentence) +- `Translation Options`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#translation-options`](non-hero-scenarios.md#translation-options) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Client Methods`: | Method | Description | + See: [`non-hero-scenarios.md#client-methods`](non-hero-scenarios.md#client-methods) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..c2b870e2 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/non-hero-scenarios.md @@ -0,0 +1,150 @@ +# azure-ai-translation-text-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Transliteration + +Convert text from one script to another: + +```python +result = client.transliterate( + body=["konnichiwa"], + language="ja", + from_script="Latn", # From Latin script + to_script="Jpan" # To Japanese script +) + +for item in result: + print(f"Transliterated: {item.text}") + print(f"Script: {item.script}") +``` + +## Dictionary Lookup + +Find alternate translations and definitions: + +```python +result = client.lookup_dictionary_entries( + body=["fly"], + from_parameter="en", + to="es" +) + +for item in result: + print(f"Source: {item.normalized_source} ({item.display_source})") + for translation in item.translations: + print(f" Translation: {translation.normalized_target}") + print(f" Part of speech: {translation.pos_tag}") + print(f" Confidence: {translation.confidence:.2f}") +``` + +## Dictionary Examples + +Get usage examples for translations: + +```python +from azure.ai.translation.text.models import DictionaryExampleTextItem + +result = client.lookup_dictionary_examples( + body=[DictionaryExampleTextItem(text="fly", translation="volar")], + from_parameter="en", + to="es" +) + +for item in result: + for example in item.examples: + print(f"Source: {example.source_prefix}{example.source_term}{example.source_suffix}") + print(f"Target: {example.target_prefix}{example.target_term}{example.target_suffix}") +``` + +## Get Supported Languages + +```python +# Get all supported languages +languages = client.get_supported_languages() + +# Translation languages +print("Translation languages:") +for code, lang in languages.translation.items(): + print(f" {code}: {lang.name} ({lang.native_name})") + +# Transliteration languages +print("\nTransliteration languages:") +for code, lang in languages.transliteration.items(): + print(f" {code}: {lang.name}") + for script in lang.scripts: + print(f" {script.code} -> {[t.code for t in script.to_scripts]}") + +# Dictionary languages +print("\nDictionary languages:") +for code, lang in languages.dictionary.items(): + print(f" {code}: {lang.name}") +``` + +## Break Sentence + +Identify sentence boundaries: + +```python +result = client.find_sentence_boundaries( + body=["Hello! How are you? I hope you are well."], + language="en" +) + +for item in result: + print(f"Sentence lengths: {item.sent_len}") +``` + +## Translation Options + +```python +result = client.translate( + body=["Hello, world!"], + to=["de"], + text_type="html", # "plain" or "html" + profanity_action="Marked", # "NoAction", "Deleted", "Marked" + profanity_marker="Asterisk", # "Asterisk", "Tag" + include_alignment=True, # Include word alignment + include_sentence_length=True # Include sentence boundaries +) + +for item in result: + translation = item.translations[0] + print(f"Translated: {translation.text}") + if translation.alignment: + print(f"Alignment: {translation.alignment.proj}") + if translation.sent_len: + print(f"Sentence lengths: {translation.sent_len.src_sent_len}") +``` + +## Async Client + +```python +from azure.ai.translation.text.aio import TextTranslationClient +from azure.identity.aio import DefaultAzureCredential + +async def translate_text(): + async with DefaultAzureCredential() as credential: + async with TextTranslationClient( + credential=credential, + endpoint=endpoint, + ) as client: + result = await client.translate( + body=["Hello, world!"], + to=["es"] + ) + print(result[0].translations[0].text) +``` + +## Client Methods + +| Method | Description | +|--------|-------------| +| `translate` | Translate text to one or more languages | +| `transliterate` | Convert text between scripts | +| `detect` | Detect language of text | +| `find_sentence_boundaries` | Identify sentence boundaries | +| `lookup_dictionary_entries` | Dictionary lookup for translations | +| `lookup_dictionary_examples` | Get usage examples | +| `get_supported_languages` | List supported languages | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/SKILL.md index ba1393b7..adaaf3c0 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/SKILL.md @@ -291,3 +291,10 @@ except HttpResponseError as e: 7. **Specify language** for localized captions 8. **Use smart_crops_aspect_ratios** matching your thumbnail requirements 9. **Cache results** when analyzing the same image multiple times + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/capabilities.md new file mode 100644 index 00000000..b24da438 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/capabilities.md @@ -0,0 +1,46 @@ +# azure-ai-vision-imageanalysis-py capability coverage + +**SDK/package**: `azure-ai-vision-imageanalysis` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Analyze Image from URL` +- `Analyze Image from File` +- `Image Caption` +- `Dense Captions (Multiple Regions)` + +## Non-hero scenarios + +- `Tags`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#tags`](non-hero-scenarios.md#tags) +- `Object Detection`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#object-detection`](non-hero-scenarios.md#object-detection) +- `OCR (Text Extraction)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#ocr-text-extraction`](non-hero-scenarios.md#ocr-text-extraction) +- `People Detection`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#people-detection`](non-hero-scenarios.md#people-detection) +- `Smart Cropping`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#smart-cropping`](non-hero-scenarios.md#smart-cropping) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Visual Features`: | Feature | Description | + See: [`non-hero-scenarios.md#visual-features`](non-hero-scenarios.md#visual-features) +- `Error Handling`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#error-handling`](non-hero-scenarios.md#error-handling) +- `Image Requirements`: - Formats: JPEG, PNG, GIF, BMP, WEBP, ICO, TIFF, MPO + See: [`non-hero-scenarios.md#image-requirements`](non-hero-scenarios.md#image-requirements) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..fe519431 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/non-hero-scenarios.md @@ -0,0 +1,137 @@ +# azure-ai-vision-imageanalysis-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Tags + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.TAGS] +) + +if result.tags: + for tag in result.tags.list: + print(f"Tag: {tag.name} (confidence: {tag.confidence:.2f})") +``` + +## Object Detection + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.OBJECTS] +) + +if result.objects: + for obj in result.objects.list: + print(f"Object: {obj.tags[0].name}") + print(f" Confidence: {obj.tags[0].confidence:.2f}") + box = obj.bounding_box + print(f" Bounding box: x={box.x}, y={box.y}, w={box.width}, h={box.height}") +``` + +## OCR (Text Extraction) + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.READ] +) + +if result.read: + for block in result.read.blocks: + for line in block.lines: + print(f"Line: {line.text}") + print(f" Bounding polygon: {line.bounding_polygon}") + + # Word-level details + for word in line.words: + print(f" Word: {word.text} (confidence: {word.confidence:.2f})") +``` + +## People Detection + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.PEOPLE] +) + +if result.people: + for person in result.people.list: + print(f"Person detected:") + print(f" Confidence: {person.confidence:.2f}") + box = person.bounding_box + print(f" Bounding box: x={box.x}, y={box.y}, w={box.width}, h={box.height}") +``` + +## Smart Cropping + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.SMART_CROPS], + smart_crops_aspect_ratios=[0.9, 1.33, 1.78] # Portrait, 4:3, 16:9 +) + +if result.smart_crops: + for crop in result.smart_crops.list: + print(f"Aspect ratio: {crop.aspect_ratio}") + box = crop.bounding_box + print(f" Crop region: x={box.x}, y={box.y}, w={box.width}, h={box.height}") +``` + +## Async Client + +```python +from azure.ai.vision.imageanalysis.aio import ImageAnalysisClient +from azure.identity.aio import DefaultAzureCredential + +async def analyze_image(): + async with DefaultAzureCredential() as credential: + async with ImageAnalysisClient( + endpoint=endpoint, + credential=credential + ) as client: + result = await client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.CAPTION] + ) + print(result.caption.text) +``` + +## Visual Features + +| Feature | Description | +|---------|-------------| +| `CAPTION` | Single sentence describing the image | +| `DENSE_CAPTIONS` | Captions for multiple regions | +| `TAGS` | Content tags (objects, scenes, actions) | +| `OBJECTS` | Object detection with bounding boxes | +| `READ` | OCR text extraction | +| `PEOPLE` | People detection with bounding boxes | +| `SMART_CROPS` | Suggested crop regions for thumbnails | + +## Error Handling + +```python +from azure.core.exceptions import HttpResponseError + +try: + result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.CAPTION] + ) +except HttpResponseError as e: + print(f"Status code: {e.status_code}") + print(f"Reason: {e.reason}") + print(f"Message: {e.error.message}") +``` + +## Image Requirements + +- Formats: JPEG, PNG, GIF, BMP, WEBP, ICO, TIFF, MPO +- Max size: 20 MB +- Dimensions: 50x50 to 16000x16000 pixels diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py/SKILL.md index 1a3f3aa1..512d0686 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py/SKILL.md @@ -329,7 +329,7 @@ except ConnectionError as e: ## Best Practices -1. **This SDK is async-only; use `azure.ai.voicelive.aio` throughout.** Do not try to pair it with sync clients from other Azure SDKs in the same call path — keep the whole request path async. +1. **This SDK is async-only; use the `.aio` namespace throughout.** Do not try to pair it with sync clients from other Azure SDKs in the same call path — keep the whole request path async. 2. **Always use context managers for clients and async credentials.** Wrap every connection in `async with connect(...) as conn:`. For async `DefaultAzureCredential` from `azure.identity.aio`, also use `async with credential:` so tokens and transports are cleaned up. ## References diff --git a/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/SKILL.md index 9fe7feb9..bd07f043 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/SKILL.md @@ -256,3 +256,10 @@ async def main(): 7. **Use Entra ID** instead of connection strings in production 8. **Refresh settings periodically** in long-running applications 9. **Use feature flags** for gradual rollouts and A/B testing + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/capabilities.md new file mode 100644 index 00000000..9b454b1c --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/capabilities.md @@ -0,0 +1,34 @@ +# azure-appconfiguration-py capability coverage + +**SDK/package**: `azure-appconfiguration` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Configuration Settings` +- `List Settings` +- `Feature Flags` +- `Read-Only Settings` + +## Non-hero scenarios + +- `Snapshots`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#snapshots`](non-hero-scenarios.md#snapshots) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Client Operations`: | Operation | Description | + See: [`non-hero-scenarios.md#client-operations`](non-hero-scenarios.md#client-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..fe023531 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/non-hero-scenarios.md @@ -0,0 +1,60 @@ +# azure-appconfiguration-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Snapshots + +### Create Snapshot + +```python +from azure.appconfiguration import ConfigurationSnapshot, ConfigurationSettingFilter + +snapshot = ConfigurationSnapshot( + name="v1-snapshot", + filters=[ + ConfigurationSettingFilter(key="app:*", label="production") + ] +) + +created = client.begin_create_snapshot( + name="v1-snapshot", + snapshot=snapshot +).result() +``` + +### List Snapshot Settings + +```python +settings = client.list_configuration_settings( + snapshot_name="v1-snapshot" +) +``` + +## Async Client + +```python +from azure.appconfiguration.aio import AzureAppConfigurationClient +from azure.identity.aio import DefaultAzureCredential + +async def main(): + async with DefaultAzureCredential() as credential: + async with AzureAppConfigurationClient( + base_url=endpoint, + credential=credential + ) as client: + setting = await client.get_configuration_setting(key="app:message") + print(setting.value) +``` + +## Client Operations + +| Operation | Description | +|-----------|-------------| +| `get_configuration_setting` | Get single setting | +| `set_configuration_setting` | Create or update setting | +| `delete_configuration_setting` | Delete setting | +| `list_configuration_settings` | List with filters | +| `set_read_only` | Lock/unlock setting | +| `begin_create_snapshot` | Create point-in-time snapshot | +| `list_snapshots` | List all snapshots | diff --git a/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/SKILL.md index e1a8305c..fb6ab982 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/SKILL.md @@ -273,3 +273,10 @@ for manifest in client.list_manifest_properties("my-image"): 7. **Use async client** for high-throughput operations 8. **Order by last_updated** to find recent/old images 9. **Check manifest.tags** before deleting to avoid removing tagged images + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/capabilities.md new file mode 100644 index 00000000..bea1d77b --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/capabilities.md @@ -0,0 +1,38 @@ +# azure-containerregistry-py capability coverage + +**SDK/package**: `azure-containerregistry` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `List Repositories` +- `Repository Operations` +- `List Tags` +- `Manifest Operations` + +## Non-hero scenarios + +- `Tag Operations`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#tag-operations`](non-hero-scenarios.md#tag-operations) +- `Upload and Download Artifacts`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#upload-and-download-artifacts`](non-hero-scenarios.md#upload-and-download-artifacts) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Clean Up Old Images`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#clean-up-old-images`](non-hero-scenarios.md#clean-up-old-images) +- `Client Operations`: | Operation | Description | + See: [`non-hero-scenarios.md#client-operations`](non-hero-scenarios.md#client-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..daa07c57 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/non-hero-scenarios.md @@ -0,0 +1,80 @@ +# azure-containerregistry-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Tag Operations + +### Get Tag Properties + +```python +tag = client.get_tag_properties("my-image", "latest") +print(f"Digest: {tag.digest}") +print(f"Created: {tag.created_on}") +``` + +### Delete Tag + +```python +client.delete_tag("my-image", "old-tag") +``` + +## Upload and Download Artifacts + +```python +from azure.containerregistry import ContainerRegistryClient + +with ContainerRegistryClient(endpoint, DefaultAzureCredential()) as client: + # Download manifest + manifest = client.download_manifest("my-image", "latest") + print(f"Media type: {manifest.media_type}") + print(f"Digest: {manifest.digest}") + + # Download blob + blob = client.download_blob("my-image", "sha256:abc123...") + with open("layer.tar.gz", "wb") as f: + for chunk in blob: + f.write(chunk) +``` + +## Async Client + +```python +from azure.containerregistry.aio import ContainerRegistryClient +from azure.identity.aio import DefaultAzureCredential + +async def list_repos(): + async with DefaultAzureCredential() as credential: + async with ContainerRegistryClient(endpoint, credential) as client: + async for repo in client.list_repository_names(): + print(repo) +``` + +## Clean Up Old Images + +```python +from datetime import datetime, timedelta, timezone + +cutoff = datetime.now(timezone.utc) - timedelta(days=30) + +for manifest in client.list_manifest_properties("my-image"): + if manifest.last_updated_on < cutoff and not manifest.tags: + print(f"Deleting {manifest.digest}") + client.delete_manifest("my-image", manifest.digest) +``` + +## Client Operations + +| Operation | Description | +|-----------|-------------| +| `list_repository_names` | List all repositories | +| `get_repository_properties` | Get repository metadata | +| `delete_repository` | Delete repository and all images | +| `list_tag_properties` | List tags in repository | +| `get_tag_properties` | Get tag metadata | +| `delete_tag` | Delete specific tag | +| `list_manifest_properties` | List manifests in repository | +| `get_manifest_properties` | Get manifest metadata | +| `delete_manifest` | Delete manifest by digest | +| `download_manifest` | Download manifest content | +| `download_blob` | Download layer blob | diff --git a/.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py/SKILL.md index 9fcb9385..66973512 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py/SKILL.md @@ -223,7 +223,7 @@ async def test_get_project_by_id_returns_project(mock_cosmos_container): ## Best Practices -1. **This skill uses async throughout (`azure.cosmos.aio`); do not mix with the sync `azure.cosmos` client.** Keep the whole FastAPI request path async — don't pair sync Cosmos calls with async handlers. +1. **Pick sync OR async and stay consistent.** Do not mix `azure.xxx` sync clients with `azure.xxx.aio` async clients in the same call path. Choose one mode per module. 2. **Always use context managers for clients and async credentials.** Wrap the client in `async with CosmosClient(...) as client:` (or manage its lifetime via FastAPI lifespan and close it explicitly). For async `DefaultAzureCredential` from `azure.identity.aio`, also use `async with credential:` so tokens and transports are cleaned up. ## Reference Files diff --git a/.github/plugins/azure-sdk-python/skills/azure-cosmos-py/scripts/setup_cosmos_container.py b/.github/plugins/azure-sdk-python/skills/azure-cosmos-py/scripts/setup_cosmos_container.py index 88cb5d06..d9baedf1 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-cosmos-py/scripts/setup_cosmos_container.py +++ b/.github/plugins/azure-sdk-python/skills/azure-cosmos-py/scripts/setup_cosmos_container.py @@ -31,7 +31,7 @@ def get_cosmos_client() -> CosmosClient: endpoint = os.environ.get("COSMOS_ENDPOINT") if not endpoint: raise ValueError("COSMOS_ENDPOINT environment variable required") - + # Try key auth first, fall back to DefaultAzureCredential key = os.environ.get("COSMOS_KEY") if key: @@ -44,33 +44,33 @@ def get_cosmos_client() -> CosmosClient: def create_indexing_policy( include_paths: list[str] | None = None, exclude_paths: list[str] | None = None, - composite_indexes: list[list[dict]] | None = None + composite_indexes: list[list[dict]] | None = None, ) -> dict[str, Any]: """Build an indexing policy.""" policy = { "indexingMode": "consistent", "automatic": True, "includedPaths": [], - "excludedPaths": [] + "excludedPaths": [], } - + # Include paths (default: all) if include_paths: policy["includedPaths"] = [{"path": p} for p in include_paths] else: policy["includedPaths"] = [{"path": "/*"}] - + # Exclude paths if exclude_paths: policy["excludedPaths"] = [{"path": p} for p in exclude_paths] - + # Always exclude _etag - policy["excludedPaths"].append({"path": "/_etag/?")}) - + policy["excludedPaths"].append({"path": "/_etag/?"}) + # Composite indexes for ORDER BY on multiple fields if composite_indexes: policy["compositeIndexes"] = composite_indexes - + return policy @@ -81,10 +81,10 @@ def create_container( partition_key_paths: list[str], throughput: int | None = None, ttl: int | None = None, - indexing_policy: dict | None = None + indexing_policy: dict | None = None, ) -> dict[str, Any]: """Create or update a Cosmos DB container.""" - + # Get or create database try: database = client.create_database_if_not_exists(id=database_id) @@ -92,41 +92,37 @@ def create_container( except CosmosHttpResponseError as e: print(f"Error creating database: {e.message}") raise - + # Build partition key if len(partition_key_paths) == 1: partition_key = PartitionKey(path=partition_key_paths[0]) else: # Hierarchical partition key partition_key = PartitionKey(path=partition_key_paths) - + # Container properties - container_props = { - "id": container_id, - "partition_key": partition_key - } - + container_props = {"id": container_id, "partition_key": partition_key} + # Add TTL if specified if ttl is not None: container_props["default_time_to_live"] = ttl - + # Add indexing policy if specified if indexing_policy: container_props["indexing_policy"] = indexing_policy - + # Create container try: if throughput: container = database.create_container_if_not_exists( - **container_props, - offer_throughput=throughput + **container_props, offer_throughput=throughput ) else: container = database.create_container_if_not_exists(**container_props) - + print(f"Container: {container_id}") print(f"Partition key: {partition_key_paths}") - + except CosmosHttpResponseError as e: if e.status_code == 409: print(f"Container {container_id} already exists") @@ -134,16 +130,16 @@ def create_container( else: print(f"Error creating container: {e.message}") raise - + # Get container properties properties = container.read() - + return { "database": database_id, "container": container_id, "partition_key": partition_key_paths, "self_link": properties.get("_self"), - "resource_id": properties.get("_rid") + "resource_id": properties.get("_rid"), } @@ -151,14 +147,14 @@ def show_container_info(client: CosmosClient, database_id: str, container_id: st """Display detailed container information.""" database = client.get_database_client(database_id) container = database.get_container_client(container_id) - + properties = container.read() - + print("\n=== Container Information ===") print(f"Database: {database_id}") print(f"Container: {container_id}") print(f"Partition Key: {properties.get('partitionKey', {}).get('paths', [])}") - + # TTL ttl = properties.get("defaultTtl") if ttl == -1: @@ -167,25 +163,29 @@ def show_container_info(client: CosmosClient, database_id: str, container_id: st print(f"TTL: {ttl} seconds") else: print("TTL: Disabled") - + # Indexing policy index_policy = properties.get("indexingPolicy", {}) print(f"Indexing Mode: {index_policy.get('indexingMode', 'consistent')}") - + # Throughput try: offer = container.read_offer() print(f"Throughput: {offer.offer_throughput} RU/s") if offer.properties.get("content", {}).get("offerAutopilotSettings"): - max_throughput = offer.properties["content"]["offerAutopilotSettings"]["maxThroughput"] + max_throughput = offer.properties["content"]["offerAutopilotSettings"][ + "maxThroughput" + ] print(f"Autoscale Max: {max_throughput} RU/s") except Exception: print("Throughput: Serverless or database-level") - + # Item count (approximate) try: query = "SELECT VALUE COUNT(1) FROM c" - count = list(container.query_items(query=query, enable_cross_partition_query=True))[0] + count = list( + container.query_items(query=query, enable_cross_partition_query=True) + )[0] print(f"Item Count: ~{count}") except Exception: print("Item Count: Unable to retrieve") @@ -195,84 +195,77 @@ def main(): parser = argparse.ArgumentParser( description="Create and configure Cosmos DB containers", formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=__doc__ - ) - - parser.add_argument( - "--database", "-d", - required=True, - help="Database ID" - ) - parser.add_argument( - "--container", "-c", - required=True, - help="Container ID" + epilog=__doc__, ) + + parser.add_argument("--database", "-d", required=True, help="Database ID") + parser.add_argument("--container", "-c", required=True, help="Container ID") parser.add_argument( - "--partition-key", "-pk", + "--partition-key", + "-pk", nargs="+", required=True, - help="Partition key path(s). Use multiple for hierarchical keys." + help="Partition key path(s). Use multiple for hierarchical keys.", ) parser.add_argument( - "--throughput", "-t", + "--throughput", + "-t", type=int, - help="Provisioned throughput in RU/s (omit for serverless)" + help="Provisioned throughput in RU/s (omit for serverless)", ) parser.add_argument( "--serverless", action="store_true", - help="Use serverless mode (no throughput provisioning)" + help="Use serverless mode (no throughput provisioning)", ) parser.add_argument( - "--ttl", - type=int, - help="Default TTL in seconds (-1 for per-item TTL)" + "--ttl", type=int, help="Default TTL in seconds (-1 for per-item TTL)" ) parser.add_argument( "--exclude-paths", nargs="+", - help="Paths to exclude from indexing (e.g., /large_field/*)" + help="Paths to exclude from indexing (e.g., /large_field/*)", ) parser.add_argument( "--composite-index", action="append", nargs="+", metavar="PATH", - help="Composite index paths (can specify multiple times). Format: --composite-index /field1 /field2" + help="Composite index paths (can specify multiple times). Format: --composite-index /field1 /field2", ) parser.add_argument( "--info", action="store_true", - help="Show container information instead of creating" + help="Show container information instead of creating", ) parser.add_argument( - "--output", "-o", + "--output", + "-o", choices=["json", "text"], default="text", - help="Output format (default: text)" + help="Output format (default: text)", ) - + args = parser.parse_args() - + # Validate arguments if args.throughput and args.serverless: print("Error: Cannot specify both --throughput and --serverless") sys.exit(1) - + # Ensure partition key paths start with / partition_keys = [] for pk in args.partition_key: if not pk.startswith("/"): pk = f"/{pk}" partition_keys.append(pk) - + try: client = get_cosmos_client() except ValueError as e: print(f"Error: {e}") sys.exit(1) - + # Show info mode if args.info: try: @@ -281,7 +274,7 @@ def main(): print(f"Error: {e.message}") sys.exit(1) return - + # Build indexing policy indexing_policy = None if args.exclude_paths or args.composite_index: @@ -291,12 +284,11 @@ def main(): [{"path": p, "order": "ascending"} for p in index_paths] for index_paths in args.composite_index ] - + indexing_policy = create_indexing_policy( - exclude_paths=args.exclude_paths, - composite_indexes=composite_indexes + exclude_paths=args.exclude_paths, composite_indexes=composite_indexes ) - + # Create container try: result = create_container( @@ -306,12 +298,12 @@ def main(): partition_key_paths=partition_keys, throughput=args.throughput if not args.serverless else None, ttl=args.ttl, - indexing_policy=indexing_policy + indexing_policy=indexing_policy, ) except CosmosHttpResponseError as e: print(f"Error: {e.message}") sys.exit(1) - + # Output result if args.output == "json": print(json.dumps(result, indent=2)) @@ -326,7 +318,7 @@ def main(): print("Throughput: Serverless") if args.ttl: print(f"TTL: {args.ttl} seconds") - + print("\nContainer ready for use!") diff --git a/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/SKILL.md index ef513ce5..6376581b 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/SKILL.md @@ -269,3 +269,10 @@ asyncio.run(table_operations()) 8. **Use parameterized queries** to prevent injection 9. **Keep entities small** — max 1MB per entity 10. **Use async client** for high-throughput scenarios + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/capabilities.md new file mode 100644 index 00000000..efb547ee --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/capabilities.md @@ -0,0 +1,34 @@ +# azure-data-tables-py capability coverage + +**SDK/package**: `azure-data-tables` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Client Types` +- `Table Operations` +- `Entity Operations` +- `Query Entities` + +## Non-hero scenarios + +- `Batch Operations`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#batch-operations`](non-hero-scenarios.md#batch-operations) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Data Types`: | Python Type | Table Storage Type | + See: [`non-hero-scenarios.md#data-types`](non-hero-scenarios.md#data-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..00a4e48b --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/non-hero-scenarios.md @@ -0,0 +1,62 @@ +# azure-data-tables-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Batch Operations + +```python +from azure.data.tables import TableTransactionError + +# Batch operations (same partition only!) +operations = [ + ("create", {"PartitionKey": "batch", "RowKey": "1", "data": "first"}), + ("create", {"PartitionKey": "batch", "RowKey": "2", "data": "second"}), + ("upsert", {"PartitionKey": "batch", "RowKey": "3", "data": "third"}), +] + +try: + table_client.submit_transaction(operations) +except TableTransactionError as e: + print(f"Transaction failed: {e}") +``` + +## Async Client + +```python +from azure.data.tables.aio import TableServiceClient, TableClient +from azure.identity.aio import DefaultAzureCredential + +async def table_operations(): + async with DefaultAzureCredential() as credential: + async with TableClient( + endpoint="https://.table.core.windows.net", + table_name="mytable", + credential=credential + ) as client: + # Create + await client.create_entity(entity={ + "PartitionKey": "async", + "RowKey": "1", + "data": "test" + }) + + # Query + async for entity in client.query_entities("PartitionKey eq 'async'"): + print(entity) + +import asyncio +asyncio.run(table_operations()) +``` + +## Data Types + +| Python Type | Table Storage Type | +|-------------|-------------------| +| `str` | String | +| `int` | Int64 | +| `float` | Double | +| `bool` | Boolean | +| `datetime` | DateTime | +| `bytes` | Binary | +| `UUID` | Guid | diff --git a/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/SKILL.md index 907ee84b..bf34e1da 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/SKILL.md @@ -193,3 +193,10 @@ with EventGridPublisherClient( 7. **Use async client** for high-throughput scenarios 8. **Handle retries** — Event Grid has built-in retry 9. **Set appropriate event types** for routing and filtering + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/capabilities.md new file mode 100644 index 00000000..5c18e7d9 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/capabilities.md @@ -0,0 +1,32 @@ +# azure-eventgrid-py capability coverage + +**SDK/package**: `azure-eventgrid` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Event Types` +- `Publish CloudEvents` +- `Publish EventGridEvents` +- `Event Properties` + +## Non-hero scenarios + +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Namespace Topics (Event Grid Namespaces)`: For Event Grid Namespaces (pull delivery): + See: [`non-hero-scenarios.md#namespace-topics-event-grid-namespaces`](non-hero-scenarios.md#namespace-topics-event-grid-namespaces) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..a74c845f --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/non-hero-scenarios.md @@ -0,0 +1,47 @@ +# azure-eventgrid-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Async Client + +```python +from azure.eventgrid.aio import EventGridPublisherClient +from azure.identity.aio import DefaultAzureCredential + +async def publish_events(): + credential = DefaultAzureCredential() + + async with EventGridPublisherClient(endpoint, credential) as client: + event = CloudEvent( + type="MyApp.Events.Test", + source="/myapp", + data={"message": "hello"} + ) + await client.send(event) + +import asyncio +asyncio.run(publish_events()) +``` + +## Namespace Topics (Event Grid Namespaces) + +For Event Grid Namespaces (pull delivery): + +```python +from azure.eventgrid import EventGridPublisherClient +from azure.identity import DefaultAzureCredential + +# Namespace endpoint (different from custom topic) +namespace_endpoint = "https://..eventgrid.azure.net" +topic_name = "my-topic" + +with EventGridPublisherClient( + endpoint=namespace_endpoint, + credential=DefaultAzureCredential() +) as client: + client.send( + event, + namespace_topic=topic_name + ) +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-identity-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-identity-py/SKILL.md index 37e6a034..f187f844 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-identity-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-identity-py/SKILL.md @@ -62,7 +62,20 @@ AZURE_CLIENT_ID= AZURE_TOKEN_CREDENTIALS=dev|prod| # Optional, restricts DAC chain ``` -## DefaultAzureCredential +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **Prefer `DefaultAzureCredential`.** It works locally (Azure CLI / VS Code / Developer CLI) and in Azure (managed identity, workload identity) with no code change. Avoid connection strings, account/API keys — they bypass Entra audit and rotation. +> - Local dev: `DefaultAzureCredential` works as-is. +> - Production: set `AZURE_TOKEN_CREDENTIALS=prod` (or `AZURE_TOKEN_CREDENTIALS=`) to constrain the credential chain to production-safe credentials. +> 2. **Wrap credentials and clients in context managers** when they own token caches / transports: +> - Sync: `with DefaultAzureCredential() as credential:` +> - Async: `async with DefaultAzureCredential() as credential:` (from `azure.identity.aio`) +> +> Snippets may abbreviate this setup, but production code should always follow both rules. + +### DefaultAzureCredential The recommended credential for most scenarios. Tries multiple authentication methods in order: @@ -522,3 +535,10 @@ AZURE_LOG_LEVEL=debug | API Reference | https://learn.microsoft.com/python/api/azure-identity | | GitHub Source | https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity | | Credential Chains | https://aka.ms/azsdk/python/identity/credential-chains | + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/capabilities.md new file mode 100644 index 00000000..bfc31d6e --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/capabilities.md @@ -0,0 +1,42 @@ +# azure-identity-py capability coverage + +**SDK/package**: `azure-identity` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `get_bearer_token_provider` +- `Credential Types` +- `Specific Credential Examples` +- `Getting Tokens Directly` + +## Non-hero scenarios + +- `Async Credentials`: Async credentials are in `azure.identity.aio`. Always close them or use `async with`: + See: [`non-hero-scenarios.md#async-credentials`](non-hero-scenarios.md#async-credentials) +- `Sovereign Clouds`: Use `AzureAuthorityHosts` or the `AZURE_AUTHORITY_HOST` env var: + See: [`non-hero-scenarios.md#sovereign-clouds`](non-hero-scenarios.md#sovereign-clouds) +- `Persistent Token Caching`: Opt-in disk-based caching with `TokenCachePersistenceOptions`: + See: [`non-hero-scenarios.md#persistent-token-caching`](non-hero-scenarios.md#persistent-token-caching) +- `Multi-Tenant Support`: Allow token acquisition for additional tenants beyond the configured one: + See: [`non-hero-scenarios.md#multi-tenant-support`](non-hero-scenarios.md#multi-tenant-support) +- `Error Handling`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#error-handling`](non-hero-scenarios.md#error-handling) +- `Logging`: Enable authentication logging for debugging: + See: [`non-hero-scenarios.md#logging`](non-hero-scenarios.md#logging) +- `Credential Selection Matrix`: | Environment | Recommended Credential | + See: [`non-hero-scenarios.md#credential-selection-matrix`](non-hero-scenarios.md#credential-selection-matrix) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..e1c9ec06 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/non-hero-scenarios.md @@ -0,0 +1,129 @@ +# azure-identity-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Async Credentials + +Async credentials are in `azure.identity.aio`. Always close them or use `async with`: + +```python +from azure.identity.aio import DefaultAzureCredential +from azure.storage.blob.aio import BlobServiceClient + +async def main(): + # Preferred: use async context manager for both credential and client + async with DefaultAzureCredential() as credential: + async with BlobServiceClient( + account_url="https://.blob.core.windows.net", + credential=credential, + ) as client: + # ... async operations + pass +``` + +> The async `get_bearer_token_provider` is at `azure.identity.aio.get_bearer_token_provider`. + +## Sovereign Clouds + +Use `AzureAuthorityHosts` or the `AZURE_AUTHORITY_HOST` env var: + +```python +from azure.identity import DefaultAzureCredential, AzureAuthorityHosts + +# Azure Government +credential = DefaultAzureCredential(authority=AzureAuthorityHosts.AZURE_GOVERNMENT) + +# Azure China +credential = DefaultAzureCredential(authority=AzureAuthorityHosts.AZURE_CHINA) +``` + +| Constant | Authority | +|----------|-----------| +| `AzureAuthorityHosts.AZURE_PUBLIC_CLOUD` | `login.microsoftonline.com` (default) | +| `AzureAuthorityHosts.AZURE_GOVERNMENT` | `login.microsoftonline.us` | +| `AzureAuthorityHosts.AZURE_CHINA` | `login.chinacloudapi.cn` | + +## Persistent Token Caching + +Opt-in disk-based caching with `TokenCachePersistenceOptions`: + +```python +from azure.identity import DefaultAzureCredential, TokenCachePersistenceOptions + +credential = DefaultAzureCredential( + cache_persistence_options=TokenCachePersistenceOptions() +) + +# Allow unencrypted fallback (NOT recommended for production) +credential = DefaultAzureCredential( + cache_persistence_options=TokenCachePersistenceOptions(allow_unencrypted_storage=True) +) +``` + +Storage: Windows (DPAPI), macOS (Keychain), Linux (Keyring). + +## Multi-Tenant Support + +Allow token acquisition for additional tenants beyond the configured one: + +```python +from azure.identity import ClientSecretCredential + +credential = ClientSecretCredential( + tenant_id="", + client_id="", + client_secret="", + additionally_allowed_tenants=["", "*"], # "*" allows any tenant +) +``` + +## Error Handling + +```python +from azure.identity import DefaultAzureCredential, CredentialUnavailableError +from azure.core.exceptions import ClientAuthenticationError + +with DefaultAzureCredential() as credential: + try: + token = credential.get_token("https://management.azure.com/.default") + except CredentialUnavailableError: + # No credential in the chain could attempt authentication + pass + except ClientAuthenticationError as e: + # Authentication was attempted but failed + # e.message contains details from each credential in the chain + pass +``` + +## Logging + +Enable authentication logging for debugging: + +```python +import logging + +# Enable verbose Azure Identity logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger("azure.identity") +logger.setLevel(logging.DEBUG) +``` + +```bash +# Or via environment variable +AZURE_LOG_LEVEL=debug +``` + +## Credential Selection Matrix + +| Environment | Recommended Credential | +|-------------|------------------------| +| Local Development | `DefaultAzureCredential` (uses Azure CLI) | +| Azure App Service | `DefaultAzureCredential` (uses Managed Identity) | +| Azure Functions | `DefaultAzureCredential` (uses Managed Identity) | +| Azure Kubernetes Service | `WorkloadIdentityCredential` | +| Azure VMs | `DefaultAzureCredential` (uses Managed Identity) | +| CI/CD Pipeline | `EnvironmentCredential` or `AzurePipelinesCredential` | +| Desktop App | `InteractiveBrowserCredential` | +| CLI / Headless Tool | `DeviceCodeCredential` | +| Middle-tier Service | `OnBehalfOfCredential` | diff --git a/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/SKILL.md index 20c836c7..ef10d433 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/SKILL.md @@ -272,3 +272,10 @@ except HttpResponseError as e: 8. **Use Key Vault references** in App Service/Functions config 9. **Cache secrets** appropriately to reduce API calls 10. **Use async clients** for high-throughput scenarios + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/capabilities.md new file mode 100644 index 00000000..5a008b97 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/capabilities.md @@ -0,0 +1,32 @@ +# azure-keyvault-py capability coverage + +**SDK/package**: `azure-keyvault-secrets, azure-keyvault-keys, azure-keyvault-certificates` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Secrets` +- `Keys` +- `Certificates` +- `Client Types Table` + +## Non-hero scenarios + +- `Async Clients`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-clients`](non-hero-scenarios.md#async-clients) +- `Error Handling`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#error-handling`](non-hero-scenarios.md#error-handling) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..2f1bd32f --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/non-hero-scenarios.md @@ -0,0 +1,35 @@ +# azure-keyvault-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Async Clients + +```python +from azure.identity.aio import DefaultAzureCredential +from azure.keyvault.secrets.aio import SecretClient + +async def get_secret(): + async with DefaultAzureCredential() as credential: + async with SecretClient(vault_url=vault_url, credential=credential) as client: + secret = await client.get_secret("my-secret") + print(secret.value) + +import asyncio +asyncio.run(get_secret()) +``` + +## Error Handling + +```python +from azure.core.exceptions import ResourceNotFoundError, HttpResponseError + +try: + secret = client.get_secret("nonexistent") +except ResourceNotFoundError: + print("Secret not found") +except HttpResponseError as e: + if e.status_code == 403: + print("Access denied - check RBAC permissions") + raise +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/SKILL.md index 8cdd7c32..20cf2ada 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/SKILL.md @@ -255,3 +255,10 @@ async def broadcast(): 7. **Handle reconnection** in client applications 8. **Use JSON** content type for structured data 9. **Close connections** gracefully with reasons + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/capabilities.md new file mode 100644 index 00000000..6964e572 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/capabilities.md @@ -0,0 +1,30 @@ +# azure-messaging-webpubsubservice-py capability coverage + +**SDK/package**: `azure-messaging-webpubsubservice` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Service Client (Server-Side)` +- `Client SDK (Python WebSocket Client)` +- `Async Service Client` +- `Client Operations` + +## Non-hero scenarios + +- `Operational hardening`: Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. + See: [`non-hero-scenarios.md#operational-hardening`](non-hero-scenarios.md#operational-hardening) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..56754214 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/non-hero-scenarios.md @@ -0,0 +1,8 @@ +# azure-messaging-webpubsubservice-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Operational hardening + +Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/SKILL.md index 99f050f5..97f22bbd 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/SKILL.md @@ -89,7 +89,7 @@ for api_center in api_centers: ## Register an API ```python -from azure.mgmt.apicenter.models import Api, ApiKind, LifecycleStage +from azure.mgmt.apicenter.models import Api, ApiKind, ApiProperties api = client.apis.create_or_update( resource_group_name="my-resource-group", @@ -97,22 +97,23 @@ api = client.apis.create_or_update( workspace_name="default", api_name="my-api", resource=Api( - title="My API", - description="A sample API for demonstration", - kind=ApiKind.REST, - lifecycle_stage=LifecycleStage.PRODUCTION, - terms_of_service={"url": "https://example.com/terms"}, - contacts=[{"name": "API Team", "email": "api-team@example.com"}] - ) + properties=ApiProperties( + title="My API", + description="A sample API for demonstration", + kind=ApiKind.REST, + terms_of_service={"url": "https://example.com/terms"}, + contacts=[{"name": "API Team", "email": "api-team@example.com"}], + ) + ), ) -print(f"Registered API: {api.title}") +print(f"Registered API: {api.properties.title}") ``` ## Create API Version ```python -from azure.mgmt.apicenter.models import ApiVersion, LifecycleStage +from azure.mgmt.apicenter.models import ApiVersion, ApiVersionProperties, LifecycleStage version = client.api_versions.create_or_update( resource_group_name="my-resource-group", @@ -121,18 +122,20 @@ version = client.api_versions.create_or_update( api_name="my-api", version_name="v1", resource=ApiVersion( - title="Version 1.0", - lifecycle_stage=LifecycleStage.PRODUCTION - ) + properties=ApiVersionProperties( + title="Version 1.0", + lifecycle_stage=LifecycleStage.PRODUCTION, + ) + ), ) -print(f"Created version: {version.title}") +print(f"Created version: {version.properties.title}") ``` ## Add API Definition ```python -from azure.mgmt.apicenter.models import ApiDefinition +from azure.mgmt.apicenter.models import ApiDefinition, ApiDefinitionProperties definition = client.api_definitions.create_or_update( resource_group_name="my-resource-group", @@ -142,9 +145,11 @@ definition = client.api_definitions.create_or_update( version_name="v1", definition_name="openapi", resource=ApiDefinition( - title="OpenAPI Definition", - description="OpenAPI 3.0 specification" - ) + properties=ApiDefinitionProperties( + title="OpenAPI Definition", + description="OpenAPI 3.0 specification", + ) + ), ) ``` @@ -154,7 +159,7 @@ definition = client.api_definitions.create_or_update( from azure.mgmt.apicenter.models import ApiSpecImportRequest, ApiSpecImportSourceFormat # Import from inline content -client.api_definitions.import_specification( +client.api_definitions.begin_import_specification( resource_group_name="my-resource-group", service_name="my-api-center", workspace_name="default", @@ -163,9 +168,9 @@ client.api_definitions.import_specification( definition_name="openapi", body=ApiSpecImportRequest( format=ApiSpecImportSourceFormat.INLINE, - value='{"openapi": "3.0.0", "info": {"title": "My API", "version": "1.0"}, "paths": {}}' + value='{"openapi": "3.0.0", "info": {"title": "My API", "version": "1.0"}, "paths": {}}', ) -) +).result() ``` ## List APIs @@ -184,7 +189,7 @@ for api in apis: ## Create Environment ```python -from azure.mgmt.apicenter.models import Environment, EnvironmentKind +from azure.mgmt.apicenter.models import Environment, EnvironmentKind, EnvironmentProperties environment = client.environments.create_or_update( resource_group_name="my-resource-group", @@ -192,18 +197,20 @@ environment = client.environments.create_or_update( workspace_name="default", environment_name="production", resource=Environment( - title="Production", - description="Production environment", - kind=EnvironmentKind.PRODUCTION, - server={"type": "Azure API Management", "management_portal_uri": ["https://portal.azure.com"]} - ) + properties=EnvironmentProperties( + title="Production", + description="Production environment", + kind=EnvironmentKind.PRODUCTION, + server={"type": "Azure API Management", "management_portal_uri": ["https://portal.azure.com"]}, + ) + ), ) ``` ## Create Deployment ```python -from azure.mgmt.apicenter.models import Deployment, DeploymentState +from azure.mgmt.apicenter.models import Deployment, DeploymentProperties, DeploymentState deployment = client.deployments.create_or_update( resource_group_name="my-resource-group", @@ -212,28 +219,32 @@ deployment = client.deployments.create_or_update( api_name="my-api", deployment_name="prod-deployment", resource=Deployment( - title="Production Deployment", - description="Deployed to production APIM", - environment_id="/workspaces/default/environments/production", - definition_id="/workspaces/default/apis/my-api/versions/v1/definitions/openapi", - state=DeploymentState.ACTIVE, - server={"runtime_uri": ["https://api.example.com"]} - ) + properties=DeploymentProperties( + title="Production Deployment", + description="Deployed to production APIM", + environment_id="/workspaces/default/environments/production", + definition_id="/workspaces/default/apis/my-api/versions/v1/definitions/openapi", + state=DeploymentState.ACTIVE, + server={"runtime_uri": ["https://api.example.com"]}, + ) + ), ) ``` ## Define Custom Metadata ```python -from azure.mgmt.apicenter.models import MetadataSchema +from azure.mgmt.apicenter.models import MetadataSchema, MetadataSchemaProperties metadata = client.metadata_schemas.create_or_update( resource_group_name="my-resource-group", service_name="my-api-center", metadata_schema_name="data-classification", resource=MetadataSchema( - schema='{"type": "string", "title": "Data Classification", "enum": ["public", "internal", "confidential"]}' - ) + properties=MetadataSchemaProperties( + schema='{"type": "string", "title": "Data Classification", "enum": ["public", "internal", "confidential"]}' + ) + ), ) ``` @@ -266,3 +277,10 @@ metadata = client.metadata_schemas.create_or_update( 6. **Import specifications** to enable API analysis and linting 7. **Use lifecycle stages** to track API maturity 8. **Add contacts** for API ownership and support + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/capabilities.md new file mode 100644 index 00000000..cca21654 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/capabilities.md @@ -0,0 +1,44 @@ +# azure-mgmt-apicenter-py capability coverage + +**SDK/package**: `azure-mgmt-apicenter` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Create API Center` +- `List API Centers` +- `Register an API` +- `Create API Version` + +## Non-hero scenarios + +- `Add API Definition`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#add-api-definition`](non-hero-scenarios.md#add-api-definition) +- `Import API Specification`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#import-api-specification`](non-hero-scenarios.md#import-api-specification) +- `List APIs`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-apis`](non-hero-scenarios.md#list-apis) +- `Create Environment`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-environment`](non-hero-scenarios.md#create-environment) +- `Create Deployment`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-deployment`](non-hero-scenarios.md#create-deployment) +- `Define Custom Metadata`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#define-custom-metadata`](non-hero-scenarios.md#define-custom-metadata) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) +- `Operations`: | Operation Group | Purpose | + See: [`non-hero-scenarios.md#operations`](non-hero-scenarios.md#operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..1fb05fd8 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/non-hero-scenarios.md @@ -0,0 +1,139 @@ +# azure-mgmt-apicenter-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Add API Definition + +```python +from azure.mgmt.apicenter.models import ApiDefinition, ApiDefinitionProperties + +definition = client.api_definitions.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1", + definition_name="openapi", + resource=ApiDefinition( + properties=ApiDefinitionProperties( + title="OpenAPI Definition", + description="OpenAPI 3.0 specification", + ) + ), +) +``` + +## Import API Specification + +```python +from azure.mgmt.apicenter.models import ApiSpecImportRequest, ApiSpecImportSourceFormat + +# Import from inline content +client.api_definitions.begin_import_specification( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1", + definition_name="openapi", + body=ApiSpecImportRequest( + format=ApiSpecImportSourceFormat.INLINE, + value='{"openapi": "3.0.0", "info": {"title": "My API", "version": "1.0"}, "paths": {}}', + ) +).result() +``` + +## List APIs + +```python +apis = client.apis.list( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default" +) + +for api in apis: + print(f"{api.name}: {api.title} ({api.kind})") +``` + +## Create Environment + +```python +from azure.mgmt.apicenter.models import Environment, EnvironmentKind, EnvironmentProperties + +environment = client.environments.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + environment_name="production", + resource=Environment( + properties=EnvironmentProperties( + title="Production", + description="Production environment", + kind=EnvironmentKind.PRODUCTION, + server={"type": "Azure API Management", "management_portal_uri": ["https://portal.azure.com"]}, + ) + ), +) +``` + +## Create Deployment + +```python +from azure.mgmt.apicenter.models import Deployment, DeploymentProperties, DeploymentState + +deployment = client.deployments.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + deployment_name="prod-deployment", + resource=Deployment( + properties=DeploymentProperties( + title="Production Deployment", + description="Deployed to production APIM", + environment_id="/workspaces/default/environments/production", + definition_id="/workspaces/default/apis/my-api/versions/v1/definitions/openapi", + state=DeploymentState.ACTIVE, + server={"runtime_uri": ["https://api.example.com"]}, + ) + ), +) +``` + +## Define Custom Metadata + +```python +from azure.mgmt.apicenter.models import MetadataSchema, MetadataSchemaProperties + +metadata = client.metadata_schemas.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + metadata_schema_name="data-classification", + resource=MetadataSchema( + properties=MetadataSchemaProperties( + schema='{"type": "string", "title": "Data Classification", "enum": ["public", "internal", "confidential"]}' + ) + ), +) +``` + +## Client Types + +| Client | Purpose | +|--------|---------| +| `ApiCenterMgmtClient` | Main client for all operations | + +## Operations + +| Operation Group | Purpose | +|----------------|---------| +| `services` | API Center service management | +| `workspaces` | Workspace management | +| `apis` | API registration and management | +| `api_versions` | API version management | +| `api_definitions` | API definition management | +| `deployments` | Deployment tracking | +| `environments` | Environment management | +| `metadata_schemas` | Custom metadata definitions | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/SKILL.md index 23cccf5a..ddec4896 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/SKILL.md @@ -302,3 +302,10 @@ user = client.user.create_or_update( 6. **Enable Application Insights** for monitoring 7. **Use backends** to abstract backend services 8. **Version your APIs** using APIM's versioning features + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/capabilities.md new file mode 100644 index 00000000..a74cb701 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/capabilities.md @@ -0,0 +1,44 @@ +# azure-mgmt-apimanagement-py capability coverage + +**SDK/package**: `azure-mgmt-apimanagement` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Create APIM Service` +- `Import API from OpenAPI` +- `Import API from URL` +- `List APIs` + +## Non-hero scenarios + +- `Create Product`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-product`](non-hero-scenarios.md#create-product) +- `Add API to Product`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#add-api-to-product`](non-hero-scenarios.md#add-api-to-product) +- `Create Subscription`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-subscription`](non-hero-scenarios.md#create-subscription) +- `Set API Policy`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#set-api-policy`](non-hero-scenarios.md#set-api-policy) +- `Create Named Value (Secret)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-named-value-secret`](non-hero-scenarios.md#create-named-value-secret) +- `Create Backend`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-backend`](non-hero-scenarios.md#create-backend) +- `Create User`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-user`](non-hero-scenarios.md#create-user) +- `Operation Groups`: | Group | Purpose | + See: [`non-hero-scenarios.md#operation-groups`](non-hero-scenarios.md#operation-groups) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..fa31a077 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/non-hero-scenarios.md @@ -0,0 +1,156 @@ +# azure-mgmt-apimanagement-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Create Product + +```python +from azure.mgmt.apimanagement.models import ProductContract + +product = client.product.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + product_id="premium", + parameters=ProductContract( + display_name="Premium", + description="Premium tier with unlimited access", + subscription_required=True, + approval_required=False, + state="published" + ) +) + +print(f"Created product: {product.display_name}") +``` + +## Add API to Product + +```python +client.product_api.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + product_id="premium", + api_id="my-api" +) +``` + +## Create Subscription + +```python +from azure.mgmt.apimanagement.models import SubscriptionCreateParameters + +subscription = client.subscription.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + sid="my-subscription", + parameters=SubscriptionCreateParameters( + display_name="My Subscription", + scope=f"/products/premium", + state="active" + ) +) + +print(f"Subscription key: {subscription.primary_key}") +``` + +## Set API Policy + +```python +from azure.mgmt.apimanagement.models import PolicyContract + +policy_xml = """ + + + + + CustomValue + + + + + + + + +""" + +client.api_policy.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + api_id="my-api", + policy_id="policy", + parameters=PolicyContract( + value=policy_xml, + format="xml" + ) +) +``` + +## Create Named Value (Secret) + +```python +from azure.mgmt.apimanagement.models import NamedValueCreateContract + +named_value = client.named_value.begin_create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + named_value_id="backend-api-key", + parameters=NamedValueCreateContract( + display_name="Backend API Key", + value="secret-key-value", + secret=True + ) +).result() +``` + +## Create Backend + +```python +from azure.mgmt.apimanagement.models import BackendContract + +backend = client.backend.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + backend_id="my-backend", + parameters=BackendContract( + url="https://api.backend.example.com", + protocol="http", + description="My backend service" + ) +) +``` + +## Create User + +```python +from azure.mgmt.apimanagement.models import UserCreateParameters + +user = client.user.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + user_id="newuser", + parameters=UserCreateParameters( + email="user@example.com", + first_name="John", + last_name="Doe" + ) +) +``` + +## Operation Groups + +| Group | Purpose | +|-------|---------| +| `api_management_service` | APIM instance management | +| `api` | API operations | +| `api_operation` | API operation details | +| `api_policy` | API-level policies | +| `product` | Product management | +| `product_api` | Product-API associations | +| `subscription` | Subscription management | +| `user` | User management | +| `named_value` | Named values/secrets | +| `backend` | Backend services | +| `certificate` | Certificates | +| `gateway` | Self-hosted gateways | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/SKILL.md index ec8513af..4ad0b650 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/SKILL.md @@ -342,3 +342,10 @@ for conn in connections: 7. **Rotate Direct Line keys** periodically 8. **Use managed identity** when possible for bot connections 9. **Configure proper CORS** for Web Chat channel + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/capabilities.md new file mode 100644 index 00000000..9cdefab7 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/capabilities.md @@ -0,0 +1,46 @@ +# azure-mgmt-botservice-py capability coverage + +**SDK/package**: `azure-mgmt-botservice` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Create a Bot` +- `Get Bot Details` +- `List Bots in Resource Group` +- `List All Bots in Subscription` + +## Non-hero scenarios + +- `Update Bot`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#update-bot`](non-hero-scenarios.md#update-bot) +- `Delete Bot`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#delete-bot`](non-hero-scenarios.md#delete-bot) +- `Configure Channels`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#configure-channels`](non-hero-scenarios.md#configure-channels) +- `Get Channel Details`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#get-channel-details`](non-hero-scenarios.md#get-channel-details) +- `List Channel Keys`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-channel-keys`](non-hero-scenarios.md#list-channel-keys) +- `Bot Connections (OAuth)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#bot-connections-oauth`](non-hero-scenarios.md#bot-connections-oauth) +- `Client Operations`: | Operation | Method | + See: [`non-hero-scenarios.md#client-operations`](non-hero-scenarios.md#client-operations) +- `SKU Options`: | SKU | Description | + See: [`non-hero-scenarios.md#sku-options`](non-hero-scenarios.md#sku-options) +- `Channel Types`: | Channel | Class | Purpose | + See: [`non-hero-scenarios.md#channel-types`](non-hero-scenarios.md#channel-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..024f6a72 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/non-hero-scenarios.md @@ -0,0 +1,208 @@ +# azure-mgmt-botservice-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Update Bot + +```python +bot = client.bots.update( + resource_group_name=resource_group, + resource_name=bot_name, + properties=BotProperties( + display_name="Updated Bot Name", + description="Updated description" + ) +) +``` + +## Delete Bot + +```python +client.bots.delete( + resource_group_name=resource_group, + resource_name=bot_name +) +``` + +## Configure Channels + +### Add Teams Channel + +```python +from azure.mgmt.botservice.models import ( + BotChannel, + MsTeamsChannel, + MsTeamsChannelProperties +) + +channel = client.channels.create( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="MsTeamsChannel", + parameters=BotChannel( + location="global", + properties=MsTeamsChannel( + properties=MsTeamsChannelProperties( + is_enabled=True + ) + ) + ) +) +``` + +### Add Direct Line Channel + +```python +from azure.mgmt.botservice.models import ( + BotChannel, + DirectLineChannel, + DirectLineChannelProperties, + DirectLineSite +) + +channel = client.channels.create( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="DirectLineChannel", + parameters=BotChannel( + location="global", + properties=DirectLineChannel( + properties=DirectLineChannelProperties( + sites=[ + DirectLineSite( + site_name="Default Site", + is_enabled=True, + is_v1_enabled=False, + is_v3_enabled=True + ) + ] + ) + ) + ) +) +``` + +### Add Web Chat Channel + +```python +from azure.mgmt.botservice.models import ( + BotChannel, + WebChatChannel, + WebChatChannelProperties, + WebChatSite +) + +channel = client.channels.create( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="WebChatChannel", + parameters=BotChannel( + location="global", + properties=WebChatChannel( + properties=WebChatChannelProperties( + sites=[ + WebChatSite( + site_name="Default Site", + is_enabled=True + ) + ] + ) + ) + ) +) +``` + +## Get Channel Details + +```python +channel = client.channels.get( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="DirectLineChannel" +) +``` + +## List Channel Keys + +```python +keys = client.channels.list_with_keys( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="DirectLineChannel" +) + +# Access Direct Line keys +if hasattr(keys.properties, 'properties'): + for site in keys.properties.properties.sites: + print(f"Site: {site.site_name}") + print(f"Key: {site.key}") +``` + +## Bot Connections (OAuth) + +### Create Connection Setting + +```python +from azure.mgmt.botservice.models import ( + ConnectionSetting, + ConnectionSettingProperties +) + +connection = client.bot_connection.create( + resource_group_name=resource_group, + resource_name=bot_name, + connection_name="graph-connection", + parameters=ConnectionSetting( + location="global", + properties=ConnectionSettingProperties( + client_id="", + client_secret="", + scopes="User.Read", + service_provider_id="" + ) + ) +) +``` + +### List Connections + +```python +connections = client.bot_connection.list_by_bot_service( + resource_group_name=resource_group, + resource_name=bot_name +) + +for conn in connections: + print(f"Connection: {conn.name}") +``` + +## Client Operations + +| Operation | Method | +|-----------|--------| +| `client.bots` | Bot CRUD operations | +| `client.channels` | Channel configuration | +| `client.bot_connection` | OAuth connection settings | +| `client.direct_line` | Direct Line channel operations | +| `client.email` | Email channel operations | +| `client.operations` | Available operations | +| `client.host_settings` | Host settings operations | + +## SKU Options + +| SKU | Description | +|-----|-------------| +| `F0` | Free tier (limited messages) | +| `S1` | Standard tier (unlimited messages) | + +## Channel Types + +| Channel | Class | Purpose | +|---------|-------|---------| +| `MsTeamsChannel` | Microsoft Teams | Teams integration | +| `DirectLineChannel` | Direct Line | Custom client integration | +| `WebChatChannel` | Web Chat | Embeddable web widget | +| `SlackChannel` | Slack | Slack workspace integration | +| `FacebookChannel` | Facebook | Messenger integration | +| `EmailChannel` | Email | Email communication | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/SKILL.md index fbb6c37b..31cb7e05 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/SKILL.md @@ -280,3 +280,10 @@ capacity = poller.result() 8. **Handle LRO properly** — don't assume immediate completion 9. **Set up capacity admins** — specify users who can manage workspaces 10. **Monitor capacity usage** via Azure Monitor metrics + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/capabilities.md new file mode 100644 index 00000000..2671e9f6 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/capabilities.md @@ -0,0 +1,48 @@ +# azure-mgmt-fabric-py capability coverage + +**SDK/package**: `azure-mgmt-fabric` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Create Fabric Capacity` +- `Get Capacity Details` +- `List Capacities in Resource Group` +- `List All Capacities in Subscription` + +## Non-hero scenarios + +- `Update Capacity`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#update-capacity`](non-hero-scenarios.md#update-capacity) +- `Suspend Capacity`: Pause capacity to stop billing: + See: [`non-hero-scenarios.md#suspend-capacity`](non-hero-scenarios.md#suspend-capacity) +- `Resume Capacity`: Resume a paused capacity: + See: [`non-hero-scenarios.md#resume-capacity`](non-hero-scenarios.md#resume-capacity) +- `Delete Capacity`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#delete-capacity`](non-hero-scenarios.md#delete-capacity) +- `Check Name Availability`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#check-name-availability`](non-hero-scenarios.md#check-name-availability) +- `List Available SKUs`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-available-skus`](non-hero-scenarios.md#list-available-skus) +- `Client Operations`: | Operation | Method | + See: [`non-hero-scenarios.md#client-operations`](non-hero-scenarios.md#client-operations) +- `Fabric SKUs`: | SKU | Description | CUs | + See: [`non-hero-scenarios.md#fabric-skus`](non-hero-scenarios.md#fabric-skus) +- `Capacity States`: | State | Description | + See: [`non-hero-scenarios.md#capacity-states`](non-hero-scenarios.md#capacity-states) +- `Long-Running Operations`: All mutating operations are long-running (LRO). Use `.result()` to wait: + See: [`non-hero-scenarios.md#long-running-operations`](non-hero-scenarios.md#long-running-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..cffbf63d --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/non-hero-scenarios.md @@ -0,0 +1,142 @@ +# azure-mgmt-fabric-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Update Capacity + +```python +from azure.mgmt.fabric.models import FabricCapacityUpdate, CapacitySku + +updated = client.fabric_capacities.begin_update( + resource_group_name=resource_group, + capacity_name=capacity_name, + properties=FabricCapacityUpdate( + sku=CapacitySku( + name="F4", # Scale up + tier="Fabric" + ), + tags={"environment": "production"} + ) +).result() + +print(f"Updated SKU: {updated.sku.name}") +``` + +## Suspend Capacity + +Pause capacity to stop billing: + +```python +client.fabric_capacities.begin_suspend( + resource_group_name=resource_group, + capacity_name=capacity_name +).result() + +print("Capacity suspended") +``` + +## Resume Capacity + +Resume a paused capacity: + +```python +client.fabric_capacities.begin_resume( + resource_group_name=resource_group, + capacity_name=capacity_name +).result() + +print("Capacity resumed") +``` + +## Delete Capacity + +```python +client.fabric_capacities.begin_delete( + resource_group_name=resource_group, + capacity_name=capacity_name +).result() + +print("Capacity deleted") +``` + +## Check Name Availability + +```python +from azure.mgmt.fabric.models import CheckNameAvailabilityRequest + +result = client.fabric_capacities.check_name_availability( + location="eastus", + body=CheckNameAvailabilityRequest( + name="my-new-capacity", + type="Microsoft.Fabric/capacities" + ) +) + +if result.name_available: + print("Name is available") +else: + print(f"Name not available: {result.reason}") +``` + +## List Available SKUs + +```python +skus = client.fabric_capacities.list_skus( + resource_group_name=resource_group, + capacity_name=capacity_name +) + +for sku in skus: + print(f"SKU: {sku.name} - Tier: {sku.tier}") +``` + +## Client Operations + +| Operation | Method | +|-----------|--------| +| `client.fabric_capacities` | Capacity CRUD operations | +| `client.operations` | List available operations | + +## Fabric SKUs + +| SKU | Description | CUs | +|-----|-------------|-----| +| `F2` | Entry level | 2 Capacity Units | +| `F4` | Small | 4 Capacity Units | +| `F8` | Medium | 8 Capacity Units | +| `F16` | Large | 16 Capacity Units | +| `F32` | X-Large | 32 Capacity Units | +| `F64` | 2X-Large | 64 Capacity Units | +| `F128` | 4X-Large | 128 Capacity Units | +| `F256` | 8X-Large | 256 Capacity Units | +| `F512` | 16X-Large | 512 Capacity Units | +| `F1024` | 32X-Large | 1024 Capacity Units | +| `F2048` | 64X-Large | 2048 Capacity Units | + +## Capacity States + +| State | Description | +|-------|-------------| +| `Active` | Capacity is running | +| `Paused` | Capacity is suspended (no billing) | +| `Provisioning` | Being created | +| `Updating` | Being modified | +| `Deleting` | Being removed | +| `Failed` | Operation failed | + +## Long-Running Operations + +All mutating operations are long-running (LRO). Use `.result()` to wait: + +```python +# Synchronous wait +capacity = client.fabric_capacities.begin_create_or_update(...).result() + +# Or poll manually +poller = client.fabric_capacities.begin_create_or_update(...) +while not poller.done(): + print(f"Status: {poller.status()}") + time.sleep(5) +capacity = poller.result() +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/SKILL.md index b6146061..56f0f86d 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/SKILL.md @@ -228,3 +228,10 @@ Stream names follow patterns: 7. **Use async client** for high-throughput scenarios 8. **Batch uploads** — SDK handles batching, but send reasonable chunks 9. **Monitor ingestion** — Check Log Analytics for ingestion status + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/capabilities.md new file mode 100644 index 00000000..be938859 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/capabilities.md @@ -0,0 +1,40 @@ +# azure-monitor-ingestion-py capability coverage + +**SDK/package**: `azure-monitor-ingestion` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Upload Custom Logs` +- `Upload from JSON File` +- `Custom Error Handling` +- `Ignore Errors` + +## Non-hero scenarios + +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Sovereign Clouds`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#sovereign-clouds`](non-hero-scenarios.md#sovereign-clouds) +- `Batching Behavior`: The SDK automatically: + See: [`non-hero-scenarios.md#batching-behavior`](non-hero-scenarios.md#batching-behavior) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) +- `Key Concepts`: | Concept | Description | + See: [`non-hero-scenarios.md#key-concepts`](non-hero-scenarios.md#key-concepts) +- `DCR Stream Name Format`: Stream names follow patterns: + See: [`non-hero-scenarios.md#dcr-stream-name-format`](non-hero-scenarios.md#dcr-stream-name-format) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..620e311e --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/non-hero-scenarios.md @@ -0,0 +1,73 @@ +# azure-monitor-ingestion-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Async Client + +```python +import asyncio +from azure.monitor.ingestion.aio import LogsIngestionClient +from azure.identity.aio import DefaultAzureCredential + +async def upload_logs(): + async with LogsIngestionClient( + endpoint=endpoint, + credential=DefaultAzureCredential() + ) as client: + await client.upload( + rule_id=rule_id, + stream_name=stream_name, + logs=logs + ) + +asyncio.run(upload_logs()) +``` + +## Sovereign Clouds + +```python +from azure.identity import AzureAuthorityHosts, DefaultAzureCredential +from azure.monitor.ingestion import LogsIngestionClient + +# Azure Government +credential = DefaultAzureCredential(authority=AzureAuthorityHosts.AZURE_GOVERNMENT) +with LogsIngestionClient( + endpoint="https://example.ingest.monitor.azure.us", + credential=credential, + credential_scopes=["https://monitor.azure.us/.default"] +) as client: + # client.upload(...) + ... +``` + +## Batching Behavior + +The SDK automatically: +- Splits logs into chunks of 1MB or less +- Compresses each chunk with gzip +- Uploads chunks in parallel + +No manual batching needed for large log sets. + +## Client Types + +| Client | Purpose | +|--------|---------| +| `LogsIngestionClient` | Sync client for uploading logs | +| `LogsIngestionClient` (aio) | Async client for uploading logs | + +## Key Concepts + +| Concept | Description | +|---------|-------------| +| **DCE** | Data Collection Endpoint — ingestion URL | +| **DCR** | Data Collection Rule — defines schema, transformations, destination | +| **Stream** | Named data flow within a DCR | +| **Custom Table** | Target table in Log Analytics (ends with `_CL`) | + +## DCR Stream Name Format + +Stream names follow patterns: +- `Custom-_CL` — For custom tables +- `Microsoft-` — For built-in tables diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/SKILL.md index bd9553c4..86f904ba 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/SKILL.md @@ -27,7 +27,16 @@ APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=xxx;IngestionEndpoint=h AZURE_TOKEN_CREDENTIALS=prod # Required only if DefaultAzureCredential is used in production ``` -> **🔑 Auth & lifecycle:** These exporters take a connection string by design, but for *AAD-authenticated ingestion* (where supported) prefer `DefaultAzureCredential` via the `credential=` parameter — see the [Azure AD Authentication](#azure-ad-authentication) section. Any Azure SDK clients you create alongside the exporter should be wrapped in `with`/`async with` blocks (and async credentials from `azure.identity.aio` likewise). +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **Prefer `DefaultAzureCredential` for ingestion auth when supported.** `APPLICATIONINSIGHTS_CONNECTION_STRING` identifies the target Application Insights resource, and `credential=DefaultAzureCredential(...)` provides Microsoft Entra authentication. +> - Local dev: `DefaultAzureCredential` works as-is. +> - Production: set `AZURE_TOKEN_CREDENTIALS=prod` (or `AZURE_TOKEN_CREDENTIALS=`) to constrain the credential chain to production-safe credentials. +> 2. **Providers are not context managers.** Flush and shut down telemetry providers explicitly at process exit so buffers are exported deterministically. +> +> Snippets may abbreviate this setup, but production code should always follow both rules. ## When to Use @@ -221,10 +230,17 @@ exporter = AzureMonitorTraceExporter( ## Best Practices 1. **Pick sync OR async and stay consistent.** Do not mix `azure.xxx` sync clients with `azure.xxx.aio` async clients in the same call path. Choose one mode per module. -2. **Flush and shut down providers at process exit.** Call the shutdown/flush APIs (e.g. `tracer_provider.shutdown()`, `meter_provider.shutdown()`, `logger_provider.shutdown()`) at process exit to flush telemetry before the process terminates. +2. **Call `provider.shutdown()` / `flush()` at process exit to flush telemetry — providers are not context managers.** 3. **Use BatchSpanProcessor** for production (not SimpleSpanProcessor) 4. **Use ApplicationInsightsSampler** for consistent sampling across services 5. **Enable offline storage** for reliability in production 6. **Use Microsoft Entra authentication** instead of instrumentation keys 7. **Set export intervals** appropriate for your workload 8. **Use the distro** (`azure-monitor-opentelemetry`) unless you need custom pipelines + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/capabilities.md new file mode 100644 index 00000000..6cddf922 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/capabilities.md @@ -0,0 +1,42 @@ +# azure-monitor-opentelemetry-exporter-py capability coverage + +**SDK/package**: `azure-monitor-opentelemetry-exporter` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Trace Exporter` +- `Metric Exporter` +- `Log Exporter` +- `From Environment Variable` + +## Non-hero scenarios + +- `Azure AD Authentication`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#azure-ad-authentication`](non-hero-scenarios.md#azure-ad-authentication) +- `Sampling`: Use `ApplicationInsightsSampler` for consistent sampling: + See: [`non-hero-scenarios.md#sampling`](non-hero-scenarios.md#sampling) +- `Offline Storage`: Configure offline storage for retry: + See: [`non-hero-scenarios.md#offline-storage`](non-hero-scenarios.md#offline-storage) +- `Disable Offline Storage`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#disable-offline-storage`](non-hero-scenarios.md#disable-offline-storage) +- `Sovereign Clouds`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#sovereign-clouds`](non-hero-scenarios.md#sovereign-clouds) +- `Exporter Types`: | Exporter | Telemetry Type | Application Insights Table | + See: [`non-hero-scenarios.md#exporter-types`](non-hero-scenarios.md#exporter-types) +- `Configuration Options`: | Parameter | Description | Default | + See: [`non-hero-scenarios.md#configuration-options`](non-hero-scenarios.md#configuration-options) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..566af556 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/non-hero-scenarios.md @@ -0,0 +1,91 @@ +# azure-monitor-opentelemetry-exporter-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Azure AD Authentication + +```python +from azure.identity import DefaultAzureCredential, ManagedIdentityCredential +from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter + +# Local dev: DefaultAzureCredential. Production: set AZURE_TOKEN_CREDENTIALS=prod or AZURE_TOKEN_CREDENTIALS= +credential = DefaultAzureCredential(require_envvar=True) +# Or use a specific credential directly in production: +# See https://learn.microsoft.com/python/api/overview/azure/identity-readme?view=azure-python#credential-classes +# credential = ManagedIdentityCredential() + +exporter = AzureMonitorTraceExporter( + credential=credential +) +``` + +## Sampling + +Use `ApplicationInsightsSampler` for consistent sampling: + +```python +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.sampling import ParentBasedTraceIdRatio +from azure.monitor.opentelemetry.exporter import ApplicationInsightsSampler + +# Sample 10% of traces +sampler = ApplicationInsightsSampler(sampling_ratio=0.1) + +trace.set_tracer_provider(TracerProvider(sampler=sampler)) +``` + +## Offline Storage + +Configure offline storage for retry: + +```python +from azure.identity import DefaultAzureCredential +from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter + +exporter = AzureMonitorTraceExporter( + credential=DefaultAzureCredential(), + storage_directory="/path/to/storage", # Custom storage path + disable_offline_storage=False # Enable retry (default) +) +``` + +## Disable Offline Storage + +```python +exporter = AzureMonitorTraceExporter( + credential=DefaultAzureCredential(), + disable_offline_storage=True # No retry on failure +) +``` + +## Sovereign Clouds + +```python +from azure.identity import AzureAuthorityHosts, DefaultAzureCredential +from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter + +# Azure Government +credential = DefaultAzureCredential(authority=AzureAuthorityHosts.AZURE_GOVERNMENT) +exporter = AzureMonitorTraceExporter( + connection_string="InstrumentationKey=xxx;IngestionEndpoint=https://xxx.in.applicationinsights.azure.us/", + credential=credential +) +``` + +## Exporter Types + +| Exporter | Telemetry Type | Application Insights Table | +|----------|---------------|---------------------------| +| `AzureMonitorTraceExporter` | Traces/Spans | requests, dependencies, exceptions | +| `AzureMonitorMetricExporter` | Metrics | customMetrics, performanceCounters | +| `AzureMonitorLogExporter` | Logs | traces, customEvents | + +## Configuration Options + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `connection_string` | Application Insights connection string | From env var | +| `credential` | Azure credential for AAD auth | None | +| `disable_offline_storage` | Disable retry storage | False | +| `storage_directory` | Custom storage path | Temp directory | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/SKILL.md index 00f1d753..ef0b2e0f 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/SKILL.md @@ -27,7 +27,16 @@ APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=xxx;IngestionEndpoint=h AZURE_TOKEN_CREDENTIALS=prod # Required only if DefaultAzureCredential is used in production ``` -> **🔑 Auth & lifecycle:** This distro is configured with a connection string by design, but for *AAD-authenticated ingestion* (where supported) prefer `DefaultAzureCredential` via the `credential=` parameter — see the [Azure AD Authentication](#azure-ad-authentication) section. Any Azure SDK clients you create alongside the exporter should be wrapped in `with`/`async with` blocks (and async credentials from `azure.identity.aio` likewise). +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **Prefer `DefaultAzureCredential` for ingestion auth when supported.** `APPLICATIONINSIGHTS_CONNECTION_STRING` identifies the target Application Insights resource, and `credential=DefaultAzureCredential(...)` provides Microsoft Entra authentication. +> - Local dev: `DefaultAzureCredential` works as-is. +> - Production: set `AZURE_TOKEN_CREDENTIALS=prod` (or `AZURE_TOKEN_CREDENTIALS=`) to constrain the credential chain to production-safe credentials. +> 2. **Providers are not context managers.** Flush and shut down telemetry providers explicitly at process exit so buffers are exported deterministically. +> +> Snippets may abbreviate this setup, but production code should always follow both rules. ## Quick Start @@ -236,7 +245,7 @@ configure_azure_monitor( ## Best Practices 1. **Pick sync OR async and stay consistent.** Do not mix `azure.xxx` sync clients with `azure.xxx.aio` async clients in the same call path. Choose one mode per module. -2. **Flush and shut down providers at process exit.** Call the shutdown/flush APIs (e.g. `tracer_provider.shutdown()`, `meter_provider.shutdown()`, `logger_provider.shutdown()`) at process exit to flush telemetry before the process terminates. +2. **Call `provider.shutdown()` / `flush()` at process exit to flush telemetry — providers are not context managers.** 3. **Call configure_azure_monitor() early** — Before importing instrumented libraries 4. **Use environment variables** for connection string in production 5. **Set cloud role name** for multi-service applications @@ -244,3 +253,10 @@ configure_azure_monitor( 7. **Use structured logging** for better log analytics queries 8. **Add custom attributes** to spans for better debugging 9. **Use Microsoft Entra authentication** for production workloads + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/capabilities.md new file mode 100644 index 00000000..8876f7ae --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/capabilities.md @@ -0,0 +1,48 @@ +# azure-monitor-opentelemetry-py capability coverage + +**SDK/package**: `azure-monitor-opentelemetry` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Explicit Configuration` +- `With Flask` +- `With Django` +- `With FastAPI` + +## Non-hero scenarios + +- `Custom Traces`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#custom-traces`](non-hero-scenarios.md#custom-traces) +- `Custom Metrics`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#custom-metrics`](non-hero-scenarios.md#custom-metrics) +- `Custom Logs`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#custom-logs`](non-hero-scenarios.md#custom-logs) +- `Sampling`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#sampling`](non-hero-scenarios.md#sampling) +- `Cloud Role Name`: Set cloud role name for Application Map: + See: [`non-hero-scenarios.md#cloud-role-name`](non-hero-scenarios.md#cloud-role-name) +- `Disable Specific Instrumentations`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#disable-specific-instrumentations`](non-hero-scenarios.md#disable-specific-instrumentations) +- `Enable Live Metrics`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#enable-live-metrics`](non-hero-scenarios.md#enable-live-metrics) +- `Azure AD Authentication`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#azure-ad-authentication`](non-hero-scenarios.md#azure-ad-authentication) +- `Auto-Instrumentations Included`: | Library | Telemetry Type | + See: [`non-hero-scenarios.md#auto-instrumentations-included`](non-hero-scenarios.md#auto-instrumentations-included) +- `Configuration Options`: | Parameter | Description | Default | + See: [`non-hero-scenarios.md#configuration-options`](non-hero-scenarios.md#configuration-options) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..384bb1c2 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/non-hero-scenarios.md @@ -0,0 +1,136 @@ +# azure-monitor-opentelemetry-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Custom Traces + +```python +from opentelemetry import trace +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor() + +tracer = trace.get_tracer(__name__) + +with tracer.start_as_current_span("my-operation") as span: + span.set_attribute("custom.attribute", "value") + # Do work... +``` + +## Custom Metrics + +```python +from opentelemetry import metrics +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor() + +meter = metrics.get_meter(__name__) +counter = meter.create_counter("my_counter") + +counter.add(1, {"dimension": "value"}) +``` + +## Custom Logs + +```python +import logging +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor() + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +logger.info("This will appear in Application Insights") +logger.error("Errors are captured too", exc_info=True) +``` + +## Sampling + +```python +from azure.monitor.opentelemetry import configure_azure_monitor + +# Sample 10% of requests +configure_azure_monitor( + sampling_ratio=0.1 +) +``` + +## Cloud Role Name + +Set cloud role name for Application Map: + +```python +from azure.monitor.opentelemetry import configure_azure_monitor +from opentelemetry.sdk.resources import Resource, SERVICE_NAME + +configure_azure_monitor( + resource=Resource.create({SERVICE_NAME: "my-service-name"}) +) +``` + +## Disable Specific Instrumentations + +```python +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor( + instrumentations=["flask", "requests"] # Only enable these +) +``` + +## Enable Live Metrics + +```python +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor( + enable_live_metrics=True +) +``` + +## Azure AD Authentication + +```python +from azure.monitor.opentelemetry import configure_azure_monitor +from azure.identity import DefaultAzureCredential, ManagedIdentityCredential + +# Local dev: DefaultAzureCredential. Production: set AZURE_TOKEN_CREDENTIALS=prod or AZURE_TOKEN_CREDENTIALS= +credential = DefaultAzureCredential(require_envvar=True) +# Or use a specific credential directly in production: +# See https://learn.microsoft.com/python/api/overview/azure/identity-readme?view=azure-python#credential-classes +# credential = ManagedIdentityCredential() + +configure_azure_monitor( + credential=credential +) +``` + +## Auto-Instrumentations Included + +| Library | Telemetry Type | +|---------|---------------| +| Flask | Traces | +| Django | Traces | +| FastAPI | Traces | +| Requests | Traces | +| urllib3 | Traces | +| httpx | Traces | +| aiohttp | Traces | +| psycopg2 | Traces | +| pymysql | Traces | +| pymongo | Traces | +| redis | Traces | + +## Configuration Options + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `connection_string` | Application Insights connection string | From env var | +| `credential` | Azure credential for AAD auth | None | +| `sampling_ratio` | Sampling rate (0.0 to 1.0) | 1.0 | +| `resource` | OpenTelemetry Resource | Auto-detected | +| `instrumentations` | List of instrumentations to enable | All | +| `enable_live_metrics` | Enable Live Metrics stream | False | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/SKILL.md index 95defd74..c0ce6826 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/SKILL.md @@ -267,3 +267,10 @@ AppExceptions 7. **Convert to DataFrame** for easier data analysis 8. **Use aggregations** to summarize metric data 9. **Filter by dimensions** to narrow metric results + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/capabilities.md new file mode 100644 index 00000000..8cf8156e --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/capabilities.md @@ -0,0 +1,30 @@ +# azure-monitor-query-py capability coverage + +**SDK/package**: `azure-monitor-query` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Logs Query Client` +- `Metrics Query Client` +- `Async Clients` +- `Common Kusto Queries` + +## Non-hero scenarios + +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..82a76632 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/non-hero-scenarios.md @@ -0,0 +1,11 @@ +# azure-monitor-query-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Client Types + +| Client | Purpose | +|--------|---------| +| `LogsQueryClient` | Query Log Analytics workspaces | +| `MetricsQueryClient` | Query Azure Monitor metrics | diff --git a/.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py/SKILL.md index b0f96a3e..9fa73c65 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py/SKILL.md @@ -37,6 +37,17 @@ AZURE_SPEECH_ENDPOINT=https://.stt.speech.microsoft.com pip install requests ``` +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **This API currently requires key-based authentication.** `DefaultAzureCredential` / Entra ID auth isn't available for this REST path yet, so use `AZURE_SPEECH_KEY` and `AZURE_SPEECH_REGION` from environment variables (never hardcoded in source). +> 2. **Use context managers for files and HTTP resources** so file handles and network connections are released deterministically: +> - Sync: `with open(...) as f:` and (when reusing connections) `with requests.Session() as session:` +> - Async: `async with aiohttp.ClientSession() as session:` +> +> Snippets may abbreviate this setup, but production code should always follow both rules. + ## Quick Start ```python @@ -352,7 +363,7 @@ Common language codes (see [full list](https://learn.microsoft.com/azure/ai-serv ## Best Practices 1. **Pick sync OR async and stay consistent.** Do not mix `azure.xxx` sync clients with `azure.xxx.aio` async clients in the same call path. Choose one mode per module. -2. **Always use context managers for clients.** Use `with httpx.Client(...) as client:` (sync) or `async with httpx.AsyncClient(...) as client:` (async) so connections are pooled and closed deterministically. +2. **Use context managers for files and HTTP resources.** Use `with open(...) as f:` and (when reusing connections) `with requests.Session() as session:` for sync code, or `async with aiohttp.ClientSession() as session:` for async code. 3. **Use WAV PCM 16kHz mono** for best compatibility 4. **Enable chunked transfer** for lower latency 5. **Cache access tokens** for 9 minutes (valid for 10) diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/SKILL.md index 45cb8391..31cd68bb 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/SKILL.md @@ -255,3 +255,10 @@ async def download_async(): 6. **Prefer `readinto()`** over `readall()` for memory efficiency 7. **Use `walk_blobs()`** for hierarchical listing 8. **Set appropriate content types** for web-served blobs + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Capability index mapping hero flows and non-hero references. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples (metadata/properties and async patterns). | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/capabilities.md new file mode 100644 index 00000000..3a153a14 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/capabilities.md @@ -0,0 +1,32 @@ +# azure-storage-blob-py capability coverage + +**SDK/package**: `azure-storage-blob` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Client Hierarchy` +- `Core Workflow` +- `Performance Tuning` +- `SAS Tokens (User Delegation)` + +## Non-hero scenarios + +- `Blob Properties and Metadata`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#blob-properties-and-metadata`](non-hero-scenarios.md#blob-properties-and-metadata) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..8952d21a --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/non-hero-scenarios.md @@ -0,0 +1,46 @@ +# azure-storage-blob-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Blob Properties and Metadata + +```python +# Get properties +properties = blob_client.get_blob_properties() +print(f"Size: {properties.size}") +print(f"Content-Type: {properties.content_settings.content_type}") +print(f"Last modified: {properties.last_modified}") + +# Set metadata +blob_client.set_blob_metadata(metadata={"category": "logs", "year": "2024"}) + +# Set content type +from azure.storage.blob import ContentSettings +blob_client.set_http_headers( + content_settings=ContentSettings(content_type="application/json") +) +``` + +## Async Client + +```python +from azure.identity.aio import DefaultAzureCredential +from azure.storage.blob.aio import BlobServiceClient + +async def upload_async(): + async with DefaultAzureCredential() as credential: + async with BlobServiceClient(account_url, credential=credential) as client: + blob_client = client.get_blob_client("mycontainer", "sample.txt") + + with open("./file.txt", "rb") as data: + await blob_client.upload_blob(data, overwrite=True) + +# Download async +async def download_async(): + async with BlobServiceClient(account_url, credential=credential) as client: + blob_client = client.get_blob_client("mycontainer", "sample.txt") + + stream = await blob_client.download_blob() + data = await stream.readall() +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/SKILL.md index cbf9de12..bd8a9e4a 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/SKILL.md @@ -233,3 +233,10 @@ asyncio.run(datalake_operations()) 8. **Use `get_paths` with `recursive=True`** for full directory listing 9. **Set metadata** for custom file attributes 10. **Consider Blob API** for simple object storage use cases + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/capabilities.md new file mode 100644 index 00000000..b1d10159 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/capabilities.md @@ -0,0 +1,36 @@ +# azure-storage-file-datalake-py capability coverage + +**SDK/package**: `azure-storage-file-datalake` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Client Hierarchy` +- `File System Operations` +- `Directory Operations` +- `File Operations` + +## Non-hero scenarios + +- `List Contents`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-contents`](non-hero-scenarios.md#list-contents) +- `File/Directory Properties`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#filedirectory-properties`](non-hero-scenarios.md#filedirectory-properties) +- `Access Control (ACL)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#access-control-acl`](non-hero-scenarios.md#access-control-acl) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..f23a4960 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/non-hero-scenarios.md @@ -0,0 +1,77 @@ +# azure-storage-file-datalake-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## List Contents + +```python +# List paths (files and directories) +for path in file_system_client.get_paths(): + print(f"{'DIR' if path.is_directory else 'FILE'}: {path.name}") + +# List paths in directory +for path in file_system_client.get_paths(path="mydir"): + print(path.name) + +# Recursive listing +for path in file_system_client.get_paths(path="mydir", recursive=True): + print(path.name) +``` + +## File/Directory Properties + +```python +# Get properties +properties = file_client.get_file_properties() +print(f"Size: {properties.size}") +print(f"Last modified: {properties.last_modified}") + +# Set metadata +file_client.set_metadata(metadata={"processed": "true"}) +``` + +## Access Control (ACL) + +```python +# Get ACL +acl = directory_client.get_access_control() +print(f"Owner: {acl['owner']}") +print(f"Permissions: {acl['permissions']}") + +# Set ACL +directory_client.set_access_control( + owner="user-id", + permissions="rwxr-x---" +) + +# Update ACL entries +from azure.storage.filedatalake import AccessControlChangeResult +directory_client.update_access_control_recursive( + acl="user:user-id:rwx" +) +``` + +## Async Client + +```python +from azure.storage.filedatalake.aio import DataLakeServiceClient +from azure.identity.aio import DefaultAzureCredential + +async def datalake_operations(): + async with DefaultAzureCredential() as credential: + async with DataLakeServiceClient( + account_url="https://.dfs.core.windows.net", + credential=credential + ) as service_client: + file_system_client = service_client.get_file_system_client("myfilesystem") + file_client = file_system_client.get_file_client("test.txt") + + await file_client.upload_data(b"async content", overwrite=True) + + download = await file_client.download_file() + content = await download.readall() + +import asyncio +asyncio.run(datalake_operations()) +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/SKILL.md index 5200889a..5adf7ad6 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/SKILL.md @@ -243,3 +243,10 @@ async def upload_file(): 6. **Create snapshots** before major changes 7. **Set quotas** to prevent unexpected storage costs 8. **Use ranges** for partial file updates + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/capabilities.md new file mode 100644 index 00000000..6044d9b1 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/capabilities.md @@ -0,0 +1,34 @@ +# azure-storage-file-share-py capability coverage + +**SDK/package**: `azure-storage-file-share` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Share Operations` +- `Directory Operations` +- `File Operations` +- `Range Operations` + +## Non-hero scenarios + +- `Snapshot Operations`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#snapshot-operations`](non-hero-scenarios.md#snapshot-operations) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..e43eab1f --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/non-hero-scenarios.md @@ -0,0 +1,46 @@ +# azure-storage-file-share-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Snapshot Operations + +### Create Snapshot + +```python +snapshot = share_client.create_snapshot() +print(f"Snapshot: {snapshot['snapshot']}") +``` + +### Access Snapshot + +```python +snapshot_client = service.get_share_client( + "my-share", + snapshot=snapshot["snapshot"] +) +``` + +## Async Client + +```python +from azure.storage.fileshare.aio import ShareServiceClient +from azure.identity.aio import DefaultAzureCredential + +async def upload_file(): + async with DefaultAzureCredential() as credential: + async with ShareServiceClient(account_url, credential=credential) as service: + share = service.get_share_client("my-share") + file_client = share.get_file_client("test.txt") + + await file_client.upload_file("Hello!") +``` + +## Client Types + +| Client | Purpose | +|--------|---------| +| `ShareServiceClient` | Account-level operations | +| `ShareClient` | Share operations | +| `ShareDirectoryClient` | Directory operations | +| `ShareFileClient` | File operations | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/SKILL.md index 2e953c26..f8ed9816 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/SKILL.md @@ -237,3 +237,10 @@ with QueueClient( 8. **Use `peek_messages`** for monitoring without affecting queue 9. **Set `time_to_live`** to prevent stale messages 10. **Consider Service Bus** for advanced features (sessions, topics) + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/capabilities.md new file mode 100644 index 00000000..af6e9c29 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/capabilities.md @@ -0,0 +1,40 @@ +# azure-storage-queue-py capability coverage + +**SDK/package**: `azure-storage-queue` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Queue Operations` +- `Send Messages` +- `Receive Messages` +- `Peek Messages` + +## Non-hero scenarios + +- `Update Message`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#update-message`](non-hero-scenarios.md#update-message) +- `Delete Message`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#delete-message`](non-hero-scenarios.md#delete-message) +- `Clear Queue`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#clear-queue`](non-hero-scenarios.md#clear-queue) +- `Queue Properties`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#queue-properties`](non-hero-scenarios.md#queue-properties) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Base64 Encoding`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#base64-encoding`](non-hero-scenarios.md#base64-encoding) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..2d595015 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/non-hero-scenarios.md @@ -0,0 +1,101 @@ +# azure-storage-queue-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Update Message + +```python +# Extend visibility or update content +messages = queue_client.receive_messages() +for message in messages: + # Extend timeout (need more time) + queue_client.update_message( + message, + visibility_timeout=60 + ) + + # Update content and timeout + queue_client.update_message( + message, + content="Updated content", + visibility_timeout=60 + ) +``` + +## Delete Message + +```python +# Delete after successful processing +messages = queue_client.receive_messages() +for message in messages: + try: + # Process... + queue_client.delete_message(message) + except Exception: + # Message becomes visible again after timeout + pass +``` + +## Clear Queue + +```python +# Delete all messages +queue_client.clear_messages() +``` + +## Queue Properties + +```python +# Get queue properties +properties = queue_client.get_queue_properties() +print(f"Approximate message count: {properties.approximate_message_count}") + +# Set/get metadata +queue_client.set_queue_metadata(metadata={"environment": "production"}) +properties = queue_client.get_queue_properties() +print(properties.metadata) +``` + +## Async Client + +```python +from azure.storage.queue.aio import QueueServiceClient, QueueClient +from azure.identity.aio import DefaultAzureCredential + +async def queue_operations(): + credential = DefaultAzureCredential() + + async with QueueClient( + account_url="https://.queue.core.windows.net", + queue_name="myqueue", + credential=credential + ) as client: + # Send + await client.send_message("Async message") + + # Receive + async for message in client.receive_messages(): + print(message.content) + await client.delete_message(message) + +import asyncio +asyncio.run(queue_operations()) +``` + +## Base64 Encoding + +```python +from azure.storage.queue import QueueClient, BinaryBase64EncodePolicy, BinaryBase64DecodePolicy + +# For binary data +with QueueClient( + account_url=account_url, + queue_name="myqueue", + credential=credential, + message_encode_policy=BinaryBase64EncodePolicy(), + message_decode_policy=BinaryBase64DecodePolicy() +) as queue_client: + # Send bytes + queue_client.send_message(b"Binary content") +``` diff --git a/.github/plugins/azure-sdk-python/skills/fastapi-router-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/fastapi-router-py/SKILL.md index 0d6f32e6..1e966fef 100644 --- a/.github/plugins/azure-sdk-python/skills/fastapi-router-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/fastapi-router-py/SKILL.md @@ -57,5 +57,11 @@ async def list_items() -> list[Item]: ## Best Practices -1. **Pick `def` or `async def` per endpoint based on whether you call async I/O;** do not mix sync and async blocking calls in one handler. +1. **Pick `def` or `async def` per endpoint based on whether you call async I/O; do not mix sync and blocking calls in one handler.** 2. **Manage long-lived resources (DB pools, HTTP clients) in `lifespan` and inject via `Depends`;** use `with`/`async with` for per-request resources. + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | diff --git a/.github/plugins/azure-sdk-python/skills/fastapi-router-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/fastapi-router-py/references/capabilities.md new file mode 100644 index 00000000..ba5506a4 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/fastapi-router-py/references/capabilities.md @@ -0,0 +1,25 @@ +# fastapi-router-py capability coverage + +**SDK/package**: `fastapi-router-py` + +This reference captures additional non-hero capabilities and API breadth so the main `SKILL.md` can stay focused on copy/paste hero flows. + +## Hero scenarios covered in SKILL.md + +- `Quick Start` +- `Authentication Patterns` +- `Response Models` +- `HTTP Status Codes` + +## Important non-hero scenarios to include when needed + +- `Integration Steps` +- `Best Practices` + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/m365-agents-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/m365-agents-py/SKILL.md index 743e9ac1..3d583a89 100644 --- a/.github/plugins/azure-sdk-python/skills/m365-agents-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/m365-agents-py/SKILL.md @@ -54,6 +54,15 @@ COPILOTSTUDIOAGENT__TENANTID= COPILOTSTUDIOAGENT__AGENTAPPID= ``` +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **This SDK is async-first — use `async def` handlers and `async with` throughout.** +> 2. **Use explicit auth managers and context-managed network resources.** Use `MsalConnectionManager` for agent auth, and wrap per-request HTTP resources in `async with` (for example, `aiohttp.ClientSession`). +> +> Snippets may abbreviate this setup, but production code should always follow both rules. + ## Core Workflow: aiohttp-hosted AgentApplication ```python @@ -333,7 +342,7 @@ asyncio.run(main()) ## Best Practices -1. **This skill is async-first (aiohttp-based).** Use async handlers and `async with` for aiohttp sessions. +1. **This SDK is async-first — use `async def` handlers and `async with` throughout.** 2. **Always use context managers for clients and async credentials.** Wrap every client in `with Client(...) as client:` (sync) or `async with Client(...) as client:` (async). For async `DefaultAzureCredential` from `azure.identity.aio`, also use `async with credential:` so tokens and transports are cleaned up. 3. Use `microsoft_agents` import prefix (underscores, not dots). 4. Use `MemoryStorage` only for development; use BlobStorage or CosmosDB in production. @@ -352,3 +361,9 @@ asyncio.run(main()) | GitHub samples (Python) | https://github.com/microsoft/Agents-for-python | | PyPI packages | https://pypi.org/search/?q=microsoft-agents | | Integrate with Copilot Studio | https://learn.microsoft.com/en-us/microsoft-365/agents-sdk/integrate-with-mcs | + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | diff --git a/.github/plugins/azure-sdk-python/skills/m365-agents-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/m365-agents-py/references/capabilities.md new file mode 100644 index 00000000..aedf8e7c --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/m365-agents-py/references/capabilities.md @@ -0,0 +1,42 @@ +# m365-agents-py capability coverage + +**SDK/package**: `microsoft-agents-hosting-core, microsoft-agents-hosting-aiohttp, microsoft-agents-activity, microsoft-agents-authentication-msal, microsoft-agents-copilotstudio-client` + +This reference mirrors the actual capability sections in `SKILL.md` and provides concrete non-hero examples for implementation guidance. + +## Hero scenarios covered in SKILL.md + +- `Core Workflow: aiohttp-hosted AgentApplication` +- `AgentApplication Routing` +- `Streaming Responses with Azure OpenAI` +- `OAuth / Auto Sign-In` + +## Important non-hero scenarios with examples + +### `Copilot Studio Client (Direct to Engine)` + +```python +import asyncio +from msal import PublicClientApplication +from microsoft_agents.activity import ActivityTypes, load_configuration_from_env +from microsoft_agents.copilotstudio.client import ( + ConnectionSettings, + CopilotClient, +) + +# Token cache (local file for interactive flows) +class LocalTokenCache: + # See samples for full implementation + pass + +def acquire_token(settings, app_client_id, tenant_id): +# ... see SKILL.md for the full example +``` + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/pydantic-models-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/pydantic-models-py/SKILL.md index 56de5e40..2605b360 100644 --- a/.github/plugins/azure-sdk-python/skills/pydantic-models-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/pydantic-models-py/SKILL.md @@ -60,3 +60,9 @@ class MyInDB(MyResponse): 1. Create models in `src/backend/app/models/` 2. Export from `src/backend/app/models/__init__.py` 3. Add corresponding TypeScript types + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | diff --git a/.github/plugins/azure-sdk-python/skills/pydantic-models-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/pydantic-models-py/references/capabilities.md new file mode 100644 index 00000000..5ed18ff5 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/pydantic-models-py/references/capabilities.md @@ -0,0 +1,25 @@ +# pydantic-models-py capability coverage + +**SDK/package**: `pydantic-models-py` + +This reference captures additional non-hero capabilities and API breadth so the main `SKILL.md` can stay focused on copy/paste hero flows. + +## Hero scenarios covered in SKILL.md + +- `Quick Start` +- `Multi-Model Pattern` +- `camelCase Aliases` +- `Optional Update Fields` + +## Important non-hero scenarios to include when needed + +- `Database Document` +- `Integration Steps` + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/skills/agent-framework-azure-ai-py b/.github/skills/agent-framework-azure-ai-py new file mode 120000 index 00000000..0f574683 --- /dev/null +++ b/.github/skills/agent-framework-azure-ai-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/agent-framework-azure-ai-py \ No newline at end of file diff --git a/.github/skills/azure-ai-contentsafety-py b/.github/skills/azure-ai-contentsafety-py new file mode 120000 index 00000000..08ab0c46 --- /dev/null +++ b/.github/skills/azure-ai-contentsafety-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-contentsafety-py \ No newline at end of file diff --git a/.github/skills/azure-ai-contentunderstanding-py b/.github/skills/azure-ai-contentunderstanding-py new file mode 120000 index 00000000..8d4a2e04 --- /dev/null +++ b/.github/skills/azure-ai-contentunderstanding-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py \ No newline at end of file diff --git a/.github/skills/azure-ai-language-conversations-py b/.github/skills/azure-ai-language-conversations-py new file mode 120000 index 00000000..257a9470 --- /dev/null +++ b/.github/skills/azure-ai-language-conversations-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-language-conversations-py \ No newline at end of file diff --git a/.github/skills/azure-ai-ml-py b/.github/skills/azure-ai-ml-py new file mode 120000 index 00000000..eaac1e82 --- /dev/null +++ b/.github/skills/azure-ai-ml-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-ml-py \ No newline at end of file diff --git a/.github/skills/azure-ai-projects-py b/.github/skills/azure-ai-projects-py new file mode 120000 index 00000000..eaaf98bc --- /dev/null +++ b/.github/skills/azure-ai-projects-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-projects-py \ No newline at end of file diff --git a/.github/skills/azure-ai-textanalytics-py b/.github/skills/azure-ai-textanalytics-py new file mode 120000 index 00000000..650baacf --- /dev/null +++ b/.github/skills/azure-ai-textanalytics-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-textanalytics-py \ No newline at end of file diff --git a/.github/skills/azure-ai-transcription-py b/.github/skills/azure-ai-transcription-py new file mode 120000 index 00000000..f5b70e97 --- /dev/null +++ b/.github/skills/azure-ai-transcription-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-transcription-py \ No newline at end of file diff --git a/.github/skills/azure-ai-translation-document-py b/.github/skills/azure-ai-translation-document-py new file mode 120000 index 00000000..d81c331a --- /dev/null +++ b/.github/skills/azure-ai-translation-document-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-translation-document-py \ No newline at end of file diff --git a/.github/skills/azure-ai-translation-text-py b/.github/skills/azure-ai-translation-text-py new file mode 120000 index 00000000..a0d31c26 --- /dev/null +++ b/.github/skills/azure-ai-translation-text-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-translation-text-py \ No newline at end of file diff --git a/.github/skills/azure-ai-vision-imageanalysis-py b/.github/skills/azure-ai-vision-imageanalysis-py new file mode 120000 index 00000000..0cf02c59 --- /dev/null +++ b/.github/skills/azure-ai-vision-imageanalysis-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py \ No newline at end of file diff --git a/.github/skills/azure-ai-voicelive-py b/.github/skills/azure-ai-voicelive-py new file mode 120000 index 00000000..ee0d9929 --- /dev/null +++ b/.github/skills/azure-ai-voicelive-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-voicelive-py \ No newline at end of file diff --git a/.github/skills/azure-appconfiguration-py b/.github/skills/azure-appconfiguration-py new file mode 120000 index 00000000..0c8182d0 --- /dev/null +++ b/.github/skills/azure-appconfiguration-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-appconfiguration-py \ No newline at end of file diff --git a/.github/skills/azure-containerregistry-py b/.github/skills/azure-containerregistry-py new file mode 120000 index 00000000..ee9c2e53 --- /dev/null +++ b/.github/skills/azure-containerregistry-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-containerregistry-py \ No newline at end of file diff --git a/.github/skills/azure-cosmos-db-py b/.github/skills/azure-cosmos-db-py new file mode 120000 index 00000000..4b58584f --- /dev/null +++ b/.github/skills/azure-cosmos-db-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-cosmos-db-py \ No newline at end of file diff --git a/.github/skills/azure-cosmos-py b/.github/skills/azure-cosmos-py new file mode 120000 index 00000000..ab82f04f --- /dev/null +++ b/.github/skills/azure-cosmos-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-cosmos-py \ No newline at end of file diff --git a/.github/skills/azure-data-tables-py b/.github/skills/azure-data-tables-py new file mode 120000 index 00000000..a2dbba4d --- /dev/null +++ b/.github/skills/azure-data-tables-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-data-tables-py \ No newline at end of file diff --git a/.github/skills/azure-eventgrid-py b/.github/skills/azure-eventgrid-py new file mode 120000 index 00000000..d3caa828 --- /dev/null +++ b/.github/skills/azure-eventgrid-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-eventgrid-py \ No newline at end of file diff --git a/.github/skills/azure-eventhub-py b/.github/skills/azure-eventhub-py new file mode 120000 index 00000000..d2fc8221 --- /dev/null +++ b/.github/skills/azure-eventhub-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-eventhub-py \ No newline at end of file diff --git a/.github/skills/azure-identity-py b/.github/skills/azure-identity-py new file mode 120000 index 00000000..dfe20c2b --- /dev/null +++ b/.github/skills/azure-identity-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-identity-py \ No newline at end of file diff --git a/.github/skills/azure-keyvault-py b/.github/skills/azure-keyvault-py new file mode 120000 index 00000000..2335eac7 --- /dev/null +++ b/.github/skills/azure-keyvault-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-keyvault-py \ No newline at end of file diff --git a/.github/skills/azure-messaging-webpubsubservice-py b/.github/skills/azure-messaging-webpubsubservice-py new file mode 120000 index 00000000..41368c4a --- /dev/null +++ b/.github/skills/azure-messaging-webpubsubservice-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-apicenter-py b/.github/skills/azure-mgmt-apicenter-py new file mode 120000 index 00000000..446781ba --- /dev/null +++ b/.github/skills/azure-mgmt-apicenter-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-apimanagement-py b/.github/skills/azure-mgmt-apimanagement-py new file mode 120000 index 00000000..adf8958c --- /dev/null +++ b/.github/skills/azure-mgmt-apimanagement-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-botservice-py b/.github/skills/azure-mgmt-botservice-py new file mode 120000 index 00000000..84a3ac19 --- /dev/null +++ b/.github/skills/azure-mgmt-botservice-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-botservice-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-fabric-py b/.github/skills/azure-mgmt-fabric-py new file mode 120000 index 00000000..67651cbb --- /dev/null +++ b/.github/skills/azure-mgmt-fabric-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-fabric-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-ingestion-py b/.github/skills/azure-monitor-ingestion-py new file mode 120000 index 00000000..a31461c2 --- /dev/null +++ b/.github/skills/azure-monitor-ingestion-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-ingestion-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-opentelemetry-exporter-py b/.github/skills/azure-monitor-opentelemetry-exporter-py new file mode 120000 index 00000000..23b34751 --- /dev/null +++ b/.github/skills/azure-monitor-opentelemetry-exporter-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-opentelemetry-py b/.github/skills/azure-monitor-opentelemetry-py new file mode 120000 index 00000000..bcee5c56 --- /dev/null +++ b/.github/skills/azure-monitor-opentelemetry-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-query-py b/.github/skills/azure-monitor-query-py new file mode 120000 index 00000000..0f8f3bd6 --- /dev/null +++ b/.github/skills/azure-monitor-query-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-query-py \ No newline at end of file diff --git a/.github/skills/azure-search-documents-py b/.github/skills/azure-search-documents-py new file mode 120000 index 00000000..13a4ec86 --- /dev/null +++ b/.github/skills/azure-search-documents-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-search-documents-py \ No newline at end of file diff --git a/.github/skills/azure-servicebus-py b/.github/skills/azure-servicebus-py new file mode 120000 index 00000000..5275c159 --- /dev/null +++ b/.github/skills/azure-servicebus-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-servicebus-py \ No newline at end of file diff --git a/.github/skills/azure-speech-to-text-rest-py b/.github/skills/azure-speech-to-text-rest-py new file mode 120000 index 00000000..c9bc2f35 --- /dev/null +++ b/.github/skills/azure-speech-to-text-rest-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py \ No newline at end of file diff --git a/.github/skills/azure-storage-blob-py b/.github/skills/azure-storage-blob-py new file mode 120000 index 00000000..6941148a --- /dev/null +++ b/.github/skills/azure-storage-blob-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-blob-py \ No newline at end of file diff --git a/.github/skills/azure-storage-file-datalake-py b/.github/skills/azure-storage-file-datalake-py new file mode 120000 index 00000000..a4f13f7b --- /dev/null +++ b/.github/skills/azure-storage-file-datalake-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-file-datalake-py \ No newline at end of file diff --git a/.github/skills/azure-storage-file-share-py b/.github/skills/azure-storage-file-share-py new file mode 120000 index 00000000..5e1e9b27 --- /dev/null +++ b/.github/skills/azure-storage-file-share-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-file-share-py \ No newline at end of file diff --git a/.github/skills/azure-storage-queue-py b/.github/skills/azure-storage-queue-py new file mode 120000 index 00000000..3260a7ed --- /dev/null +++ b/.github/skills/azure-storage-queue-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-queue-py \ No newline at end of file diff --git a/.github/skills/fastapi-router-py b/.github/skills/fastapi-router-py new file mode 120000 index 00000000..d190edc7 --- /dev/null +++ b/.github/skills/fastapi-router-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/fastapi-router-py \ No newline at end of file diff --git a/.github/skills/m365-agents-py b/.github/skills/m365-agents-py new file mode 120000 index 00000000..a4dddc0e --- /dev/null +++ b/.github/skills/m365-agents-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/m365-agents-py \ No newline at end of file diff --git a/.github/skills/pydantic-models-py b/.github/skills/pydantic-models-py new file mode 120000 index 00000000..5f5f6459 --- /dev/null +++ b/.github/skills/pydantic-models-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/pydantic-models-py \ No newline at end of file diff --git a/.github/skills/skill-creator/SKILL.md b/.github/skills/skill-creator/SKILL.md index 83773837..337d521f 100644 --- a/.github/skills/skill-creator/SKILL.md +++ b/.github/skills/skill-creator/SKILL.md @@ -89,11 +89,11 @@ For Azure SDK skills, follow the **Skill Section Order** below. For domain skill ### Bundled Resources (Optional) -| Type | When to Include | Examples | -| ------------- | ------------------------- | -------------------------- | -| `scripts/` | Reused code patterns | Auth setup, CLI scripts | -| `references/` | Detailed patterns/schemas | API docs, migration guides | -| `assets/` | Output templates | Boilerplate code, images | +| Type | When to Include | Examples | +| ------------- | ---------------------------------------- | ---------------------------------------------------------- | +| `scripts/` | Reused code patterns | Auth setup, CLI scripts | +| `references/` | Feature deep-dives and overflow examples | `capabilities.md` index, `non-hero-scenarios.md`, API docs | +| `assets/` | Output templates | Boilerplate code, images | --- @@ -101,6 +101,85 @@ For Azure SDK skills, follow the **Skill Section Order** below. For domain skill When creating skills for Azure SDKs, follow these patterns consistently. +### Token Budget Guidelines (REQUIRED) + +Every Azure SDK skill MUST stay within these token limits: + +| Section | Target | Absolute Max | +| ----------------------------- | ---------------- | ---------------- | +| Installation + Env Vars | 100 tokens | 150 | +| Authentication & Lifecycle | 200 tokens | 300 | +| Core Workflow (1 example) | 300 tokens | 400 | +| Feature Tables | 200 tokens | 300 | +| Best Practices (6-8 items) | 200 tokens | 250 | +| References (reference/ links) | 100 tokens | 150 | +| **Total SKILL.md** | **~1100 tokens** | **~1500 tokens** | + +**Enforcement**: + +- Exceeding max limit → refactor into `/references/` subdirectories +- When approaching 500 lines → move entire sections to reference files +- Annotate with `# Token Count: ~XXXX` at top of completed skill + +--- + +### Reference Extraction Guide (REQUIRED) + +Decide what goes in SKILL.md vs. `/references/` using these signals: + +| Signal | Move to `/references/` | Keep in SKILL.md | +| -------------- | ----------------------------------- | ---------------------- | +| Use frequency | <20% of typical use | ~80%+ of workflows | +| Cognitive load | Advanced patterns, multiple options | Single happy path | +| Example length | >10 lines, multiple paths | 1-5 lines, single path | + +**Content extraction rules:** + +- **Batch operations** → `/references/batch-operations.md` +- **Error handling** (beyond try-except) → `/references/error-handling.md` +- **Performance tuning** → `/references/performance.md` +- **Alternative workflows** → `/references/workflows-comparison.md` +- **Streaming/events** → `/references/streaming.md` +- **Advanced auth** → `/references/auth-strategies.md` +- **Tool integration** → `/references/tools.md` +- **Breaking changes** → `/references/migration.md` + +**Decision:** Keep common case in SKILL.md, move edge cases to `/references/`. + +--- + +### Core Workflow Discipline (REQUIRED) + +Every Azure SDK skill must clarify which workflow(s) it documents. + +**Case 1: Single clear "core workflow"** (majority of services) + +If one pattern handles ~80% of use cases: + +1. Designate it as the core workflow +2. Show ONLY this workflow in SKILL.md (one complete, runnable example) +3. Defer alternatives to `/references/`: + - Batch operations → `/references/batch-operations.md` + - Error handling → `/references/error-handling.md` + - Performance tuning → `/references/performance.md` + - Alternative workflows → `/references/other-workflows.md` + +**Example**: Azure Key Vault Secrets (core workflow: retrieve a secret using managed identity). Alternative workflows in `/references/`: API-key auth, certificate auth, service principal auth. + +**Case 2: Multiple equally-valid "core workflows"** (e.g., authentication strategies, deployment targets) + +If no single pattern dominates: + +1. Pick the most common workflow for SKILL.md +2. Acknowledge other approaches in a `/references/workflows-comparison.md` that explains trade-offs (e.g., "Local dev uses DeveloperToolsCredential; production uses ManagedIdentityCredential") +3. Do NOT treat valid alternatives as "advanced" — they're equally valid, just different contexts + +**Example**: Azure Identity SDK has 8 credential types. Pick `DefaultAzureCredential` for SKILL.md (covers most local + production scenarios). Document `ManagedIdentityCredential`, `WorkloadIdentityCredential`, `AzureCliCredential`, etc. in `/references/credential-types.md` without ranking them. + +**Decision rule**: If you're unsure, ask: "Would a user choosing the other approach call what I wrote wrong?" If no, both are core workflows—acknowledge both with trade-offs. + +--- + ### Skill Section Order Follow this structure (based on existing Azure SDK skills): @@ -109,10 +188,10 @@ Follow this structure (based on existing Azure SDK skills): 2. **Installation** — `pip install`, `npm install`, etc. 3. **Environment Variables** — Required configuration, with an inline comment explaining when it's required . If using `DefaultAzureCredential`in production,include `AZURE_TOKEN_CREDENTIALS` (set to `prod` or ``) 4. **Authentication & Lifecycle** — Use a specific Microsoft Entra Token credential like `ManagedIdentityCredential` or `WorkloadIdentityCredential` for production. `DefaultAzureCredential` is only recommended for local development. To use DefaultAzureCredential in production, set the environment variable `AZURE_TOKEN_CREDENTIALS` to `prod` or the specific target credential. **For Python skills, this section MUST start with the standard callout block** (see [Required Authentication & Lifecycle Callout (Python)](#required-authentication--lifecycle-callout-python) below). -5. **Core Workflow** — Minimal viable example +5. **Core Workflow** — Minimal viable example (per core workflow discipline above) 6. **Feature Tables** — Clients, methods, tools 7. **Best Practices** — Numbered list -8. **Reference Links** — Table linking to `/references/*.md` +8. **Reference Links** — Table linking to `/references/*.md` (for Azure SDK skills, include `capabilities.md` + `non-hero-scenarios.md`) ### Required Authentication & Lifecycle Callout (Python) @@ -262,6 +341,97 @@ let client = BlobServiceClient::new( **Never hardcode credentials. Use environment variables.** +### Anti-Patterns: What NOT to Do (REQUIRED Reading) + +**These patterns cause bloat and inefficiency. Every skill author must review this section before writing.** + +#### Anti-Pattern 1: "Exhaustive API Reference" + +- ❌ **Don't**: List all 50 SDK methods in a feature table with code samples for every variant +- ✅ **Do**: Show 3-5 core methods in a table; link to official Azure API reference for exhaustive list +- **Token cost**: Listing all methods + examples = 400-600 tokens wasted +- **User impact**: Overwhelming cognitive load; users don't know what to use + +#### Anti-Pattern 2: "Multiple Ways to Solve One Problem" + +- ❌ **Don't**: "Here's approach A, B, C, and D to paginate results" in the main body +- ✅ **Do**: "Use `ItemPaged` for sync pagination" (primary example); link alternatives to `/references/` +- **Token cost**: Each alternate approach = 50-100 tokens; 5 approaches = skill becomes inefficient +- **User impact**: Decision paralysis; users re-read everything + +#### Anti-Pattern 3: "Beginner + Intermediate + Advanced in One Skill" + +- ❌ **Don't**: Skill that goes from "what is a client?" to "custom retry policies" to "circuit breaker patterns" +- ✅ **Do**: Core workflow covers 80% use case; advanced patterns in `/references/` +- **Token cost**: Every skill level adds 200-300 tokens; three levels = 600-900 extra tokens +- **User impact**: Experts bored, beginners overwhelmed; nobody gets what they need + +#### Anti-Pattern 4: "Restating Official Documentation" + +- ❌ **Don't**: "The CosmosClient constructor takes an endpoint (string) and credential (TokenCredential). The endpoint identifies the Azure Cosmos resource..." +- ✅ **Do**: Show code: `client = CosmosClient(endpoint, credential)`. Link to official docs: `microsoft-docs` MCP. +- **Token cost**: Verbose explanation = 50-100 tokens per parameter; large APIs waste 300+ tokens +- **User impact**: Redundant; official docs are authoritative, skill should show usage not repeat them + +#### Anti-Pattern 5: "Verbose Explanation When Example Suffices" + +- ❌ **Don't**: "To create a client, you first instantiate the class using the constructor, passing the endpoint and credential parameters. The endpoint is a string that identifies your resource..." +- ✅ **Do**: Show code immediately: `with CosmosClient(endpoint, credential) as client:` + +--- + +### Vally Efficiency Validation (REQUIRED - Phase 2) + +**During authoring, validate skill efficiency against vally framework before publication.** + +**1. Measure token count:** + +Use a token counter or model playground to measure each section. Compare to the Token Budget Guidelines targets above. If any section exceeds max, move content to `/references/`. + +**2. Run anti-pattern checklist:** + +- [ ] No exhaustive API reference (show 3-5 core methods, not 50) +- [ ] No multiple solutions to one problem in SKILL.md +- [ ] No beginner+intermediate+advanced mixed +- [ ] No restating official docs (code first, link to microsoft-docs) +- [ ] No verbose prose (examples first, minimal text) + +**3. Example count audit:** + +- [ ] 1-2 complete examples in core workflow (not 5+) +- [ ] Feature table includes 3-5 core methods (not comprehensive API) +- [ ] Max 1 example per best practice bullet + +**4. Frontmatter validation:** + +- [ ] `name` matches `.github/skills//SKILL.md` +- [ ] `description` includes trigger keywords +- [ ] `description` is ≤200 chars +- [ ] Optional `benchmark_tokens` and `benchmark_quality` included + +**4b. Authentication guidance validation** (critical for all credentials): + +- [ ] If skill uses Azure Identity credentials, verify guidance against [official credential-chains docs](https://learn.microsoft.com/en-us/azure/developer/python/sdk/authentication/credential-chains#usage-guidance-for-defaultazurecredential) +- [ ] Development guidance: OK to recommend `DefaultAzureCredential` (supports multiple dev credential types) +- [ ] Production guidance: Must NOT recommend `DefaultAzureCredential` alone; recommend specific credential (e.g., `ManagedIdentityCredential`) +- [ ] Link to `/references/auth-strategies.md` or official docs for production credential selection + +**5. Spot check:** + +- [ ] Can a user copy the core workflow and run it immediately? +- [ ] Do all examples follow best practices (context managers, appropriate credentials)? +- [ ] Are all environment variables documented? + +**Output:** After validation, annotate the skill header with measured token count: + +```markdown +# Azure Service SDK + + +``` + +--- + ### Standard Verb Patterns Azure SDKs use consistent verbs across all languages: @@ -304,15 +474,15 @@ Add both items verbatim (adapted only for language/SDK specifics) as the **first **Variants to apply when the SDK shape differs:** -| Skill type | Adjust item #1 to | Adjust item #2 to | -| ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Async-only SDK (e.g. voicelive) | "This SDK is async-only; use the `.aio` namespace throughout." | keep standard | -| Async-first framework (agent framework, m365-agents) | "This SDK is async-first — use `async def` handlers and `async with` throughout." | keep standard | -| Provider-pattern (OpenTelemetry exporters/distro) | keep standard | "Call `provider.shutdown()` / `flush()` at process exit to flush telemetry — providers are not context managers." | -| REST-over-httpx skills | keep standard | "Use `with httpx.Client(...) as client:` (sync) or `async with httpx.AsyncClient(...) as client:` (async) so connections pool and close deterministically." | -| Identity skill | keep standard | "Use credentials as context managers (`with DefaultAzureCredential() as credential:`) when they own token caches / HTTP transports you want cleaned up; for async, use `async with` on credentials from `azure.identity.aio`." | -| FastAPI (non-Azure) | "Pick `def` or `async def` per endpoint based on whether you call async I/O; do not mix sync and blocking calls in one handler." | "Manage long-lived resources (DB pools, HTTP clients) in `lifespan` and inject via `Depends`; use `with`/`async with` for per-request resources." | -| Pure model/schema skill (no I/O, e.g. pydantic) | **skip both** — not applicable | **skip** | +| Skill type | Adjust item #1 to | Adjust item #2 to | +| ----------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Async-only SDK (e.g. voicelive) | "This SDK is async-only; use the `.aio` namespace throughout." | keep standard | +| Framework guidance that is async-oriented (for example some agent frameworks) | "Use the framework's documented async patterns where required, but do not claim async is globally preferred for Azure Python SDKs." | keep standard | +| Provider-pattern (OpenTelemetry exporters/distro) | keep standard | "Call `provider.shutdown()` / `flush()` at process exit to flush telemetry — providers are not context managers." | +| REST-over-httpx skills | keep standard | "Use `with httpx.Client(...) as client:` (sync) or `async with httpx.AsyncClient(...) as client:` (async) so connections pool and close deterministically." | +| Identity skill | keep standard | "Use credentials as context managers (`with DefaultAzureCredential() as credential:`) when they own token caches / HTTP transports you want cleaned up; for async, use `async with` on credentials from `azure.identity.aio`." | +| FastAPI (non-Azure) | "Pick `def` or `async def` per endpoint based on whether you call async I/O; do not mix sync and blocking calls in one handler." | "Manage long-lived resources (DB pools, HTTP clients) in `lifespan` and inject via `Depends`; use `with`/`async with` for per-request resources." | +| Pure model/schema skill (no I/O, e.g. pydantic) | **skip both** — not applicable | **skip** | **Enforcement in code examples.** Every code example inside the skill must itself obey both rules, so the skill demonstrates what it prescribes: @@ -336,6 +506,30 @@ Add both items verbatim (adapted only for language/SDK specifics) as the **first 6. **Always verify package versions using crates.io.** Before using a package, check its version on [crates.io](https://crates.io/) to ensure you are using a stable and supported release. +### Example Effective Skills (Benchmark These) + +**Study these 4 skills for conciseness, focus, and efficiency.** Your new skills should match this profile or explicitly explain why they're larger. + +| Skill | Est. Tokens | Tokens Δ | Quality Δ | Key Pattern to Copy | +| --------------------------- | ----------- | ----------- | ---------- | --------------------------------------------------------- | +| `azure-ai-ml-py` | ~1,200 | **-39%** ✅ | Maintained | Single workflow: train + register model; concise examples | +| `azure-ai-textanalytics-py` | ~1,100 | **-39%** ✅ | Maintained | Feature table (5-7 core methods); minimal prose | +| `azure-ai-contentsafety-py` | ~1,150 | **-37%** ✅ | Maintained | Compact use-case focus; each example is 1-2 lines | +| `azure-appconfiguration-py` | ~1,180 | **-34%** ✅ | Maintained | Strategic link to references; no comprehensive API doc | + +**Why these are effective**: + +1. Stay under 1,200 tokens (core limit) +2. Show ONE canonical workflow, not 5 variants +3. Show 1-2 examples per concept, not 3-5 +4. Use tables for API summary (not prose descriptions) +5. Link to official docs via `microsoft-docs` MCP instead of duplicating +6. Move advanced patterns to `/references/` + +**Before writing your skill**: Ask "Which of these 4 is my use case most similar to?" then mirror that structure. + +--- + ### Handling Deprecated or Rebranded SDKs When an Azure SDK has been deprecated or rebranded, update skills to guide users toward the current package while maintaining backward compatibility: @@ -471,10 +665,12 @@ item = client.create_item(name="example", data={...}) ## Reference Files -| File | Contents | -| -------------------------------------------------- | ------------------------ | -| [references/tools.md](references/tools.md) | Tool integrations | -| [references/streaming.md](references/streaming.md) | Event streaming patterns | +| File | Contents | +| -------------------------------------------------------------------- | ------------------------------------------------------ | +| [references/capabilities.md](references/capabilities.md) | Capability index (hero coverage + links to deep-dives) | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Concrete non-hero examples | +| [references/tools.md](references/tools.md) | Tool integrations | +| [references/streaming.md](references/streaming.md) | Event streaming patterns | ``` --- @@ -577,17 +773,43 @@ Skills are organized by **language** and **product area** in the `skills/` direc **Write bundled resources first**, then SKILL.md. -**Frontmatter:** +**Quality assurance before finalizing:** + +1. Measure section token counts as you write (use model playground token counter) +2. Compare to Token Budget Guidelines targets +3. Validate against anti-patterns checklist (see Anti-Patterns section) +4. Extract to `/references/` if section exceeds max tokens +5. Run Vally efficiency validation checklist (see Vally Efficiency Validation) +6. Update frontmatter with `benchmark_tokens` and `benchmark_quality` metadata +7. Add token count comment to skill header for future maintenance + +**Frontmatter (Enhanced with Benchmarking Metadata):** ```yaml --- -name: skill-name-py +name: azure-service-py description: | Azure Service SDK for Python. Use for [specific features]. Triggers: "service name", "create resource", "specific operation". +benchmark_tokens: + estimated: 1180 + target: 1100 + max: 1500 +benchmark_quality: + single_core_workflow: true + examples_focused: true + no_prose_bloat: true + anti_patterns_checked: true --- ``` +**Metadata fields:** + +- `benchmark_tokens.estimated` — Actual measured token count +- `benchmark_tokens.target` — Target efficiency (typically 1100) +- `benchmark_tokens.max` — Absolute ceiling (1500; split if exceeded) +- `benchmark_quality.*` — Boolean checklist for validation + ### Step 5: Categorize with Symlinks After creating the skill in `.github/skills/`, create a symlink in the appropriate category: @@ -629,6 +851,8 @@ ls -la skills/python/foundry/agents **Location:** `tests/scenarios//acceptance-criteria.md` +> Keep acceptance criteria in the `tests/` tree (never beside `SKILL.md` inside the skill folder). + **Source materials** (in priority order): 1. Official Microsoft Learn docs (via `microsoft-docs` MCP) @@ -827,7 +1051,36 @@ For Azure SDK language skills, use official upstream source docs and examples as - Update "Best Practices" and "Reference Links" when upstream recommendations change - For Rust, if code uses `azure_core` types/imports directly, ensure `azure_core` is present in `Cargo.toml`; if only service-crate re-exports are used, direct `azure_core` dependency is optional -3. **Validate regenerated skill behavior** +### API Surface Parity Gate (required for every regenerated skill) + +Treat Microsoft Learn API reference as the contract for every snippet in the regenerated skill. + +Before finalizing any regenerated skill: + +1. Identify each SDK type/method shown in snippets (clients, operation groups, model constructors, enum members, long-running methods like `begin_*`). +2. Verify each symbol and signature against current Microsoft Learn API reference pages for that package. +3. If Learn shows a different shape (for example nested `properties=...` models, renamed methods, `begin_*` LRO methods), update the snippet to the Learn shape. +4. Re-check imports so model/client modules match Learn exactly. +5. Do not keep compatibility shortcuts that contradict Learn examples in primary snippets. + +### Scenario Coverage Gate (required for every regenerated skill, all languages) + +Regeneration is not complete when snippets compile — it is complete when the skill demonstrates real usage breadth. + +Before finalizing any regenerated skill: + +1. Identify **hero scenarios** from current Learn quickstarts/tutorials/samples for that SDK. +2. Ensure each hero scenario is represented in the skill with copy-pastable snippets (or an explicit link to a bundled reference file when too large). +3. Add/refresh test scenarios so hero flows are validated by harness patterns. +4. Add at least **one important non-hero scenario** (for example: update/patch, delete/cleanup, export/import, advanced auth mode, paging/filtering, retries/error handling, or LRO monitoring) when supported by the SDK. For Python SDKs that support both sync and async clients, present both forms with equal priority; do not treat either as universally preferred. +5. For Azure SDK skills, structure `references/` as: + - `references/capabilities.md` as a concise index (hero scenarios + links to deeper non-hero references, no historical/migration narration). + - `references/non-hero-scenarios.md` for concrete non-hero examples that are intentionally kept out of the main `SKILL.md`. + - Additional `references/*.md` files for specialized deep-dives (operation groups, tools, evaluator matrices, etc.). +6. If the SDK has broad operation-group coverage (common in management SDKs), include an operation-group table and explicitly call out which groups are covered in snippets vs. referenced only. +7. Never claim "full API surface" unless the skill genuinely demonstrates all major operation groups; otherwise state that the skill is optimized for hero workflows plus selected secondary scenarios. + +8. **Validate regenerated skill behavior** ```bash cd tests @@ -866,6 +1119,66 @@ In the PR/commit notes, include: - Which snippets/signatures were corrected - Which tests/evals were run and their outcomes +#### Python plugin batch recipe: `azure-sdk-python` + +Use this when the request is "regenerate all Python skills under azure-sdk-python." + +1. **Scope the exact targets first** + +```bash +# Canonical source of truth for Python plugin skills +ls .github/plugins/azure-sdk-python/skills/*/SKILL.md +``` + +- Treat `.github/plugins/azure-sdk-python/skills/` as canonical. +- Keep `.github/skills/` links in sync after edits (symlink check/fix step below). + +2. **For each skill, refresh from authoritative sources** + +- Always use `microsoft-docs` MCP first for current Microsoft Learn API guidance. +- Verify installed package API surface with `pip show ` before finalizing snippets. +- For Azure SDK skills, prefer package overview + official SDK repo examples. +- For non-Azure Python skills in this plugin (for example `fastapi-router-py`, `pydantic-models-py`), keep language-specific best-practice variants and skip Azure-specific auth callouts when lifecycle/auth is not applicable. + +3. **Apply Python enforcement rules consistently** + +- Keep the standard section order for Azure SDK Python skills. +- Ensure `## Authentication & Lifecycle` starts with the required callout block (verbatim) when applicable. +- Ensure every client example uses `with` / `async with` lifecycle patterns. +- Ensure `## Best Practices` starts with the two required user-facing rules (or the documented variant for async-only/provider-pattern skills). +- Ensure each regenerated Azure SDK Python skill has `references/capabilities.md` (index) and `references/non-hero-scenarios.md` (concrete non-hero examples). +- Keep existing references/assets/scripts unless stale or incorrect. + +4. **Validate all regenerated Python skills** + +```bash +# Fast frontmatter/structure validation for every Python skill +python .github/skills/skill-creator/scripts/quick_validate.py .github/plugins/azure-sdk-python/skills/ + +# Run Python skill harness in mock mode (all *-py scenarios) +cd tests +pwsh ./run-harness-by-language.ps1 -Language py -Mock +``` + +5. **Sync skill links and docs artifacts** + +```bash +# Ensure .github/skills links point at plugin canonical skills +python .github/scripts/sync_skill_links.py --plugin azure-sdk-python --check +python .github/scripts/sync_skill_links.py --plugin azure-sdk-python --apply + +# Refresh docs site data after content changes +cd docs-site && npx tsx scripts/extract-skills.ts +cd docs-site && npm run build +``` + +6. **Completion criteria for batch regeneration** + +- Every targeted `.github/plugins/azure-sdk-python/skills/*/SKILL.md` is updated or explicitly confirmed current. +- Harness mock run for `-py` skills passes without regressions. +- Skill links are in sync for `azure-sdk-python`. +- PR notes include upstream docs used, signature corrections, and validation outcomes. + --- ## Progressive Disclosure Patterns @@ -938,6 +1251,9 @@ azure-ai-agents/ | Use wrong import paths | Azure SDKs have specific module structures | | Omit sync/async + context-manager bullets from Best Practices in Python skills | End users won't follow rules that aren't written down; examples alone aren't enough | | Mix sync and async in the same Python example | Demonstrates the anti-pattern the skill is supposed to prevent | +| Ship regenerated skills with zero test scenarios | Hero workflows and regressions cannot be validated | +| Claim full API coverage from a single happy-path sample | Hides operation-group and non-hero gaps users need for production | +| Omit `references/*.md` coverage for non-hero capabilities | Forces advanced capabilities out of context and leaves API breadth undocumented | --- @@ -949,6 +1265,7 @@ Before completing a skill: - [ ] User provided SDK package name or documentation URL - [ ] Verified SDK patterns via `microsoft-docs` MCP +- [ ] Verified every snippet's API surface against current Microsoft Learn API reference pages for that SDK **Skill Creation:** @@ -956,7 +1273,12 @@ Before completing a skill: - [ ] SKILL.md under 500 lines - [ ] Authentication follows language rules (`DefaultAzureCredential` for Python/.NET/Java/TS/Go local dev; `DeveloperToolsCredential` local dev + `ManagedIdentityCredential` production for Rust) - [ ] Includes cleanup/delete in examples -- [ ] References organized by feature +- [ ] References organized by feature (`capabilities.md` index + dedicated deep-dive files) +- [ ] Hero scenarios from current Learn docs are explicitly covered in snippets and tests +- [ ] At least one high-value non-hero scenario is included (or explicitly documented as intentionally out of scope) +- [ ] For Azure SDK skills, `references/capabilities.md` indexes hero/non-hero coverage and links to dedicated non-hero docs +- [ ] For Azure SDK skills, `references/non-hero-scenarios.md` contains concrete non-hero examples distinct from hero snippets +- [ ] For broad SDKs (especially management SDKs), operation-group coverage is explicit (covered in snippets vs. reference-only) - [ ] **(Python skills only) Best Practices section contains the two user-facing rules** (sync-or-async consistency + context managers for clients and async credentials), using the variant matched to the skill type - [ ] For Rust skills: `## Best Practices` starts with cargo dependency rule + `azure_core` direct-import rule @@ -970,6 +1292,7 @@ Before completing a skill: - [ ] `tests/scenarios//acceptance-criteria.md` created with correct/incorrect patterns - [ ] `tests/scenarios//scenarios.yaml` created +- [ ] At least one hero scenario and one non-hero scenario are test-covered (when the SDK supports both) - [ ] All scenarios pass (`pnpm harness --mock`) - [ ] Import paths documented precisely diff --git a/.github/skills/skill-creator/references/azure-sdk-patterns.md b/.github/skills/skill-creator/references/azure-sdk-patterns.md index ca7922c7..dc2108d3 100644 --- a/.github/skills/skill-creator/references/azure-sdk-patterns.md +++ b/.github/skills/skill-creator/references/azure-sdk-patterns.md @@ -9,14 +9,15 @@ Reference for creating skills that teach agents to write code following official ## Table of Contents 1. [Core Principles (All Languages)](#core-principles-all-languages) -2. [Standard Naming Conventions](#standard-naming-conventions) -3. [Python Patterns](#python-patterns) -4. [.NET (C#) Patterns](#net-c-patterns) -5. [Java Patterns](#java-patterns) -6. [TypeScript/JavaScript Patterns](#typescriptjavascript-patterns) -7. [Rust Patterns](#rust-patterns) -8. [Authentication (All Languages)](#authentication-all-languages) -9. [Quick Reference Tables](#quick-reference-tables) +2. [Skill Reference Directory Pattern](#skill-reference-directory-pattern) +3. [Standard Naming Conventions](#standard-naming-conventions) +4. [Python Patterns](#python-patterns) +5. [.NET (C#) Patterns](#net-c-patterns) +6. [Java Patterns](#java-patterns) +7. [TypeScript/JavaScript Patterns](#typescriptjavascript-patterns) +8. [Rust Patterns](#rust-patterns) +9. [Authentication (All Languages)](#authentication-all-languages) +10. [Quick Reference Tables](#quick-reference-tables) --- @@ -36,6 +37,20 @@ Azure SDKs follow five design principles. Skills should reinforce these: --- +## Skill Reference Directory Pattern + +For Azure SDK skills, keep `SKILL.md` focused on hero flows and use `references/` for overflow details: + +- `references/capabilities.md` is an index only: hero scenarios covered in `SKILL.md`, non-hero scenario list, and links to deep-dive reference files. +- `references/non-hero-scenarios.md` contains concrete non-hero examples intentionally kept out of `SKILL.md`. +- Additional `references/*.md` files are optional for specialized topics (operation groups, evaluator/tool matrices, migration notes). + +Use present-tense guidance in reference files; avoid historical migration notes in user-facing capability indexes. + +For Python SDK skills that provide both sync and async clients, present both forms as first-class options. Do not encode a blanket preference for either mode in capability prioritization. + +--- + ## Standard Naming Conventions ### Namespace/Package Format diff --git a/.github/workflows/vally-evaluation.yml b/.github/workflows/run-vally-evaluations.yml similarity index 68% rename from .github/workflows/vally-evaluation.yml rename to .github/workflows/run-vally-evaluations.yml index b5975531..84650aaa 100644 --- a/.github/workflows/vally-evaluation.yml +++ b/.github/workflows/run-vally-evaluations.yml @@ -1,15 +1,15 @@ -name: Vally Evaluation +name: Vally Workflow Evaluation on: pull_request: paths: - "tests/scenarios/**/vally/**" - - ".github/workflows/vally-evaluation.yml" + - ".github/workflows/run-vally-evaluations.yml" push: branches: [main] paths: - "tests/scenarios/**/vally/**" - - ".github/workflows/vally-evaluation.yml" + - ".github/workflows/run-vally-evaluations.yml" workflow_dispatch: inputs: eval_spec: @@ -19,13 +19,21 @@ on: permissions: contents: read + # eval.yaml's `executor: copilot-sdk` makes vally spawn the real GitHub + # Copilot CLI (`@github/copilot`, pulled in transitively via the vally npm + # dependency tree -- no extra install step needed) to run each stimulus. + # That CLI authenticates via the GH_TOKEN/GITHUB_TOKEN env var, which + # requires the GITHUB_TOKEN to carry the "Copilot Requests" permission. + copilot-requests: write jobs: vally: runs-on: ubuntu-latest env: - GH_TOKEN: ${{ secrets.COPILOT_TOKEN || '' }} - + # Authenticates Copilot CLI and Vally's Copilot SDK evaluator path. + GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} + GITHUB_COPILOT_API_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@v4 with: @@ -33,14 +41,36 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "24" - name: Install Vally CLI - run: npm install -g @microsoft/vally-cli@0.6.0 + run: npm install -g @microsoft/vally-cli@0.7.0 + + - name: Install Copilot CLI + run: npm install -g @github/copilot + + - name: Install Copilot CLI + run: npm install -g @github/copilot + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true - name: Verify Vally CLI run: vally --version + - name: Verify copilot CLI + shell: bash + # env: + # Authenticates the Copilot CLI process vally spawns (see the + # copilot-requests permission note above). GH_TOKEN takes + # precedence over GITHUB_TOKEN if both are set. + # GH_TOKEN: ${{ github.token }} + run: | + copilot -p "What is the day of the week?" + - name: Resolve eval specs id: specs shell: bash @@ -86,8 +116,8 @@ jobs: else find tests/scenarios -type f \( -name 'eval.yaml' -o -name 'eval.yml' \) -path '*/vally/*' | sort -u > "$specs_file" fi - elif [ "${{ github.event_name }}" = "pull_request" ]; then - collect_from_changed_dirs "${{ github.event.pull_request.base.sha }}" "${{ github.sha }}" | sort -u > "$specs_file" + elif [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "pull_request_target" ]; then + collect_from_changed_dirs "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" | sort -u > "$specs_file" else base_sha="${{ github.event.before }}" if [ -z "$base_sha" ] || [ "$base_sha" = '0000000000000000000000000000000000000000' ]; then @@ -125,6 +155,11 @@ jobs: - name: Lint Vally eval specs if: ${{ steps.specs.outputs.has_specs == 'true' }} + # env: + # Authenticates the Copilot CLI process vally spawns (see the + # copilot-requests permission note above). GH_TOKEN takes + # precedence over GITHUB_TOKEN if both are set. + # GH_TOKEN: ${{ github.token }} shell: bash run: | set -euo pipefail @@ -138,7 +173,7 @@ jobs: done - name: Run Vally evaluations - if: ${{ steps.specs.outputs.has_specs == 'true' && env.GH_TOKEN != '' }} + if: ${{ steps.specs.outputs.has_specs == 'true' }} shell: bash run: | set -euo pipefail @@ -156,10 +191,10 @@ jobs: --grader-plugin "$GITHUB_WORKSPACE/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure" \ --junit --output-dir vally-results --workers 2 - - name: Skip eval when token is unavailable - if: ${{ steps.specs.outputs.has_specs == 'true' && env.GH_TOKEN == '' }} - run: | - echo "::warning::COPILOT_TOKEN secret is not available. Skipping vally eval run and keeping lint-only validation." + # - name: Skip eval when token is unavailable + # if: ${{ steps.specs.outputs.has_specs == 'true' && env.GH_TOKEN == '' }} + # run: | + # echo "::warning::GH_TOKEN secret is not available. Skipping vally eval run and keeping lint-only validation." - name: Upload Vally artifacts if: always() diff --git a/.github/workflows/test-harness.yml b/.github/workflows/test-harness.yml index 6dc78f91..1805c2da 100644 --- a/.github/workflows/test-harness.yml +++ b/.github/workflows/test-harness.yml @@ -24,9 +24,10 @@ jobs: cache: "pnpm" cache-dependency-path: tests/pnpm-lock.yaml - - name: Verify Rust skill links are in sync + - name: Verify Skill links are in sync run: | python .github/scripts/sync_skill_links.py --plugin azure-sdk-rust --check + python .github/scripts/sync_skill_links.py --plugin azure-sdk-python --check - name: Install dependencies working-directory: tests diff --git a/tests/.gitignore b/tests/.gitignore index 933d3cab..925d65b4 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -10,6 +10,9 @@ results.md results.json *.report.md *.report.json +vally-experiment-results/ +scenario-results/ +coverage/ # ============================================================================= # Node.js diff --git a/tests/compare-experiment.mjs b/tests/compare-experiment.mjs new file mode 100644 index 00000000..1ca0dce2 --- /dev/null +++ b/tests/compare-experiment.mjs @@ -0,0 +1,301 @@ +#!/usr/bin/env node +/** + * compare-experiment.mjs + * + * Produces a detailed A/B comparison for a Vally experiment directory. + * An experiment directory contains one sub-folder per variant, each with + * results.jsonl and run-summary.jsonl. + * + * Usage: + * node compare-experiment.mjs [--baseline ] [--skill ] + * + * Examples: + * node compare-experiment.mjs Q:\src\skills\vally-experiment-results\2026-06-23T20-24-42-839Z + * node compare-experiment.mjs . --baseline sonnet_baseline --skill sonnet_skill + * + * If --baseline / --skill are omitted the script auto-selects: + * - baseline: first variant alphabetically (or variant containing "baseline") + * - skill: second variant (or variant containing "skill") + * + * Exit codes: 0 = success, 1 = usage/parse error. + */ + +import fs from 'fs'; +import path from 'path'; + +// ── CLI ────────────────────────────────────────────────────────────────────── +const args = process.argv.slice(2); +if (args.length === 0 || args[0] === '--help' || args[0] === '-h') { + console.log('Usage: node compare-experiment.mjs [--baseline ] [--skill ]'); + process.exit(0); +} + +const experimentDir = path.resolve(args[0]); + +let baselineVariant = null; +let skillVariant = null; +for (let i = 1; i < args.length; i++) { + if (args[i] === '--baseline' && args[i + 1]) baselineVariant = args[++i]; + if (args[i] === '--skill' && args[i + 1]) skillVariant = args[++i]; +} + +// ── Discover run directories ───────────────────────────────────────────────── +if (!fs.existsSync(experimentDir)) { + console.error(`Error: directory not found: ${experimentDir}`); + process.exit(1); +} + +const runDirs = fs.readdirSync(experimentDir) + .filter(name => { + const full = path.join(experimentDir, name); + return fs.statSync(full).isDirectory() && + fs.existsSync(path.join(full, 'results.jsonl')) && + fs.existsSync(path.join(full, 'run-summary.jsonl')); + }); + +if (runDirs.length < 2) { + console.error(`Error: need at least 2 run directories with results.jsonl in ${experimentDir}`); + console.error(`Found: ${runDirs.join(', ') || '(none)'}`); + process.exit(1); +} + +// Auto-select if not specified +if (!baselineVariant || !skillVariant) { + const hasBL = runDirs.find(d => d.toLowerCase().includes('baseline')); + const hasSK = runDirs.find(d => d.toLowerCase().includes('skill')); + baselineVariant = baselineVariant ?? (hasBL || runDirs[0]); + skillVariant = skillVariant ?? (hasSK || runDirs.find(d => d !== baselineVariant) || runDirs[1]); +} + +for (const v of [baselineVariant, skillVariant]) { + if (!runDirs.includes(v)) { + console.error(`Error: variant "${v}" not found. Available: ${runDirs.join(', ')}`); + process.exit(1); + } +} + +// ── Load results ───────────────────────────────────────────────────────────── +function loadResults(variantName) { + const dir = path.join(experimentDir, variantName); + // results.jsonl may be multi-line (one record per stimulus); aggregate them + const rows = fs.readFileSync(path.join(dir, 'results.jsonl'), 'utf8') + .split('\n').filter(l => l.trim()) + .map(l => JSON.parse(l)); + const summary = JSON.parse(fs.readFileSync(path.join(dir, 'run-summary.jsonl'), 'utf8').trim()); + return { rows, summary, variantName }; +} + +const bData = loadResults(baselineVariant); +const sData = loadResults(skillVariant); + +// ── Aggregate across stimuli ───────────────────────────────────────────────── +function aggregate(data) { + const { rows, summary, variantName } = data; + let totalScore = 0, passCount = 0, errorCount = 0, skillActivations = 0; + let turns = 0, toolCalls = 0, wallTimeMs = 0, durationMs = 0, llmCalls = 0; + let inputTokens = 0, outputTokens = 0, totalTokens = 0; + let cacheReadTokens = 0, cacheWriteTokens = 0; + + const graderMap = {}; // name -> {bScores, sScores} — built later per-side + + for (const row of rows) { + const m = row.trajectory?.metrics ?? {}; + const tok = m.tokenUsage ?? {}; + const gr = row.gradeResult ?? {}; + + totalScore += gr.score ?? 0; + passCount += gr.passed ? 1 : 0; + errorCount += m.errorCount ?? 0; + skillActivations += m.skillActivationCount ?? 0; + turns += m.turnCount ?? 0; + toolCalls += m.toolCallCount ?? 0; + wallTimeMs += m.wallTimeMs ?? 0; + durationMs += row.durationMs ?? 0; + llmCalls += tok.callCount ?? 0; + inputTokens += tok.inputTokens ?? 0; + outputTokens += tok.outputTokens ?? 0; + totalTokens += tok.totalTokens ?? 0; + cacheReadTokens += tok.cacheReadTokens ?? 0; + cacheWriteTokens += tok.cacheWriteTokens ?? 0; + + // Collect per-grader details (top-level detail list only) + for (const g of (gr.details ?? [])) { + if (!graderMap[g.name]) graderMap[g.name] = { scores: [], passes: [] }; + graderMap[g.name].scores.push(g.score ?? null); + graderMap[g.name].passes.push(g.passed ?? null); + } + } + + const n = rows.length || 1; + return { + variantName, + stimulusCount: rows.length, + experiment: summary.experiment, + evalFile: summary.evalFile, + model: summary.resolvedDefaults?.model ?? '?', + skills: summary.resolvedEnvironment?.skills ?? [], + // quality + avgScore: totalScore / n, + passRate: passCount / n, + errorCount, + skillActivations, + // efficiency + turns, llmCalls, toolCalls, + wallTimeSec: (wallTimeMs / 1000), + durationSec: (durationMs / 1000), + // tokens + inputTokens, outputTokens, totalTokens, + cacheReadTokens, cacheWriteTokens, + netNewInputTokens: inputTokens - cacheReadTokens, + cacheReadRate: inputTokens > 0 ? (cacheReadTokens / inputTokens) : 0, + cacheShareOfTotal: totalTokens > 0 ? (cacheReadTokens / totalTokens) : 0, + cacheWriteRate: inputTokens > 0 ? (cacheWriteTokens / inputTokens) : 0, + cacheWriteShareOfTotal: totalTokens > 0 ? (cacheWriteTokens / totalTokens) : 0, + readToWriteEfficiency: cacheWriteTokens > 0 ? (cacheReadTokens / cacheWriteTokens) : null, + // graders + graderMap, + }; +} + +const bS = aggregate(bData); +const sS = aggregate(sData); + +// ── Helpers ─────────────────────────────────────────────────────────────────── +const W = 44; // label column width + +function fmt(v, decimals = 0) { + if (v == null || v === '') return 'N/A'; + if (typeof v === 'boolean') return v ? 'true' : 'false'; + const n = parseFloat(v); + if (isNaN(n)) return String(v); + return decimals > 0 ? n.toFixed(decimals) : n.toLocaleString(); +} + +function pct(a, b) { + if (!b) return ' N/A'; + const d = (a - b) / b * 100; + const s = (d >= 0 ? '+' : '') + d.toFixed(1) + '%'; + return s.padStart(8); +} + +function row(label, bv, sv, lowerBetter, decimals = 0) { + const bvN = parseFloat(bv); + const svN = parseFloat(sv); + let deltaStr = ' '; + let winnerStr = ''; + if (!isNaN(bvN) && !isNaN(svN)) { + deltaStr = pct(svN, bvN); + if (lowerBetter !== null && bvN !== svN) { + winnerStr = lowerBetter + ? (svN < bvN ? ' ← skill' : ' ← baseline') + : (svN > bvN ? ' ← skill' : ' ← baseline'); + } + } + console.log( + label.slice(0, W).padEnd(W), + fmt(bv, decimals).padStart(14), + fmt(sv, decimals).padStart(14), + deltaStr + winnerStr + ); +} + +function divider(title = '') { + if (title) { + console.log('\n' + title); + console.log('─'.repeat(W + 42)); + } else { + console.log('─'.repeat(W + 42)); + } +} + +// ── Header ──────────────────────────────────────────────────────────────────── +console.log('\n' + '═'.repeat(W + 42)); +console.log('EXPERIMENT COMPARISON'); +console.log('═'.repeat(W + 42)); +console.log(`Experiment : ${bS.experiment}`); +console.log(`Eval file : ${bS.evalFile}`); +console.log(`Model : ${bS.model}`); +console.log(`Directory : ${experimentDir}`); +console.log(`Stimuli : ${bS.stimulusCount}`); +console.log(`Baseline : ${bS.variantName} (skills: ${bS.skills.length === 0 ? 'none' : bS.skills.map(s => path.basename(s)).join(', ')})`); +console.log(`Skill : ${sS.variantName} (skills: ${sS.skills.length === 0 ? 'none' : sS.skills.map(s => path.basename(s)).join(', ')})`); +console.log(''); +console.log('Metric'.padEnd(W), 'Baseline'.padStart(14), 'Skill'.padStart(14), ' Delta'); +divider(); + +// ── Quality ─────────────────────────────────────────────────────────────────── +row('Overall score (0-1)', bS.avgScore, sS.avgScore, false, 4); +row('Pass rate (≥threshold)', bS.passRate, sS.passRate, false, 4); +row('Error count', bS.errorCount, sS.errorCount, true); +row('Skill activations', bS.skillActivations, sS.skillActivations, false); + +// ── Efficiency ──────────────────────────────────────────────────────────────── +divider('EFFICIENCY (lower is better)'); +row('Turns (LLM turns)', bS.turns, sS.turns, true); +row('LLM API calls', bS.llmCalls, sS.llmCalls, true); +row('Tool calls total', bS.toolCalls, sS.toolCalls, true); +row('Wall time (sec)', bS.wallTimeSec, sS.wallTimeSec, true, 1); +row('Total duration (sec)', bS.durationSec, sS.durationSec, true, 1); + +// ── Token cost ──────────────────────────────────────────────────────────────── +divider('TOKEN COST (lower is better)'); +row('Total tokens', bS.totalTokens, sS.totalTokens, true); +row('Input tokens', bS.inputTokens, sS.inputTokens, true); +row('Output tokens', bS.outputTokens, sS.outputTokens, true); +row('Cache read tokens (info)', bS.cacheReadTokens, sS.cacheReadTokens, null); +row('Cache write tokens (info)',bS.cacheWriteTokens, sS.cacheWriteTokens, null); +row('Net new input tokens', bS.netNewInputTokens, sS.netNewInputTokens, true); +row('Cache utilization rate', bS.cacheReadRate, sS.cacheReadRate, false, 4); +row('Cache share of total', bS.cacheShareOfTotal, sS.cacheShareOfTotal, false, 4); +row('Cache write rate', bS.cacheWriteRate, sS.cacheWriteRate, true, 4); +row('Cache write share total', bS.cacheWriteShareOfTotal, sS.cacheWriteShareOfTotal, true, 4); +row('Read/write efficiency', bS.readToWriteEfficiency, sS.readToWriteEfficiency, false, 4); + +// ── Per-grader ──────────────────────────────────────────────────────────────── +divider('PER-GRADER SCORES'); +console.log('Grader'.padEnd(W), 'Baseline'.padStart(14), 'Skill'.padStart(14), ' Delta'); +divider(); + +const allNames = [...new Set([...Object.keys(bS.graderMap), ...Object.keys(sS.graderMap)])]; +for (const name of allNames) { + const bg = bS.graderMap[name]; + const sg = sS.graderMap[name]; + const bScore = bg ? bg.scores.reduce((a, v) => a + (v ?? 0), 0) / bg.scores.length : null; + const sScore = sg ? sg.scores.reduce((a, v) => a + (v ?? 0), 0) / sg.scores.length : null; + const bPass = bg ? (bg.passes.every(p => p) ? 'PASS' : 'FAIL') : '----'; + const sPass = sg ? (sg.passes.every(p => p) ? 'PASS' : 'FAIL') : '----'; + + const bvN = bScore ?? NaN; + const svN = sScore ?? NaN; + let deltaStr = ' ', winnerStr = ''; + if (!isNaN(bvN) && !isNaN(svN)) { + const d = svN - bvN; + deltaStr = ((d >= 0 ? '+' : '') + d.toFixed(3)).padStart(8); + winnerStr = d > 0 ? ' ← skill↑' : d < 0 ? ' ← baseline↑' : ' tie'; + } + + const bCell = `(${bPass}) ${bScore != null ? bScore.toFixed(3) : 'N/A'}`; + const sCell = `(${sPass}) ${sScore != null ? sScore.toFixed(3) : 'N/A'}`; + console.log(name.slice(0, W - 1).padEnd(W), bCell.padStart(14), sCell.padStart(14), deltaStr + winnerStr); +} + +// ── Verdict ─────────────────────────────────────────────────────────────────── +divider(); +const tokenDelta = pct(sS.totalTokens, bS.totalTokens).trim(); +const timeDelta = pct(sS.wallTimeSec, bS.wallTimeSec).trim(); +const scoreDelta = pct(sS.avgScore, bS.avgScore).trim(); +const bothPass = bS.passRate >= 1 && sS.passRate >= 1; + +console.log('\nVERDICT'); +console.log(`Both runs passed threshold : ${bothPass}`); +console.log(`Score delta : ${scoreDelta}`); +console.log(`Token delta : ${tokenDelta}`); +console.log(`Wall-time delta : ${timeDelta}`); +if (bothPass) { + const skillBetter = sS.totalTokens < bS.totalTokens && sS.wallTimeSec < bS.wallTimeSec; + const msg = skillBetter + ? `✅ Skill variant passes with ${tokenDelta} fewer tokens and ${timeDelta} less wall time → SKILL IS BETTER` + : `⚠️ Skill variant does not uniformly reduce both tokens and time — review per-metric deltas above.`; + console.log(msg); +} +console.log(''); diff --git a/tests/generate-skill-experiments.ps1 b/tests/generate-skill-experiments.ps1 new file mode 100644 index 00000000..e2df3ed1 --- /dev/null +++ b/tests/generate-skill-experiments.ps1 @@ -0,0 +1,231 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS +Generate vally skill effectiveness experiments for specified skills. + +.DESCRIPTION +Creates skill_effectiveness_experiment.yaml and skill_effectiveness_eval.yaml files +for all matching skills in the scenarios directory. Each experiment compares +performance with and without the skill context (baseline vs. skill variants). + +Use -SkillPattern to filter by language (e.g., '*-py' for Python, '*-rs' for Rust) +or by service (e.g., 'azure-cosmos*' for Cosmos DB). + +.PARAMETER ScenariosRoot +Root directory containing skill scenario subdirectories. +Defaults to: /scenarios + +.PARAMETER SkillPattern +Filter which skills to process. Supports wildcards. +Defaults to '*-py' (all Python skills). +Examples: -SkillPattern '*-rust', -SkillPattern 'azure-ai-*' + +.PARAMETER DryRun +If set, shows what would be created without actually creating files. + +.PARAMETER Force +If set, overwrites existing experiment files. Otherwise skips existing files. + +.EXAMPLE +./generate-skill-experiments.ps1 +# Generates experiments for all Python skills with default settings + +.EXAMPLE +./generate-skill-experiments.ps1 -SkillPattern '*-rust' +# Generates experiments for all Rust skills + +.EXAMPLE +./generate-skill-experiments.ps1 -SkillPattern 'azure-cosmos*' -DryRun -Verbose +# Preview Cosmos DB experiment files that would be created + +.EXAMPLE +./generate-skill-experiments.ps1 -Force +# Regenerate all experiment files, overwriting existing ones +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ScenariosRoot = (Join-Path $PSScriptRoot "scenarios"), + + [Parameter(Mandatory = $false)] + [string]$SkillPattern = "*-py", + + [Parameter(Mandatory = $false)] + [switch]$DryRun, + + [Parameter(Mandatory = $false)] + [switch]$Force +) + +$ErrorActionPreference = "Stop" + +function New-ExperimentFile { + param( + [Parameter(Mandatory = $true)] + [string]$SkillName, + + [Parameter(Mandatory = $true)] + [string]$SkillPath, + + [Parameter(Mandatory = $true)] + [string]$OutputPath, + + [Parameter(Mandatory = $false)] + [switch]$DryRun + ) + + $experimentName = "$SkillName-skill-experiment" + $content = @" +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: $experimentName +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - $SkillPath + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] +"@ + + if ($DryRun) { + Write-Verbose "Would create: $OutputPath" + Write-Verbose $content + } + else { + Set-Content -Path $OutputPath -Value $content -Encoding UTF8 + Write-Information "✓ Created: $OutputPath" + } +} + +function Copy-EvalFile { + param( + [Parameter(Mandatory = $true)] + [string]$SourcePath, + + [Parameter(Mandatory = $true)] + [string]$DestPath, + + [Parameter(Mandatory = $false)] + [switch]$DryRun + ) + + if (Test-Path $SourcePath) { + if ($DryRun) { + Write-Verbose "Would copy: $SourcePath -> $DestPath" + } + else { + Copy-Item -Path $SourcePath -Destination $DestPath -Force + Write-Information "✓ Copied: $DestPath" + } + return $true + } + return $false +} + +# Find all matching skill scenario directories +$skillDirs = Get-ChildItem -Path $ScenariosRoot -Directory -Filter $SkillPattern | +Sort-Object Name + +Write-Information "Found $($skillDirs.Count) skills matching pattern: $SkillPattern" + +$created = 0 +$skipped = 0 +$failed = 0 + +foreach ($skillDir in $skillDirs) { + $skillName = $skillDir.Name + $vallyDir = Join-Path $skillDir.FullName "vally" + + if (-not (Test-Path $vallyDir)) { + Write-Warning "No vally directory found for $skillName (expected: $vallyDir)" + $skipped++ + continue + } + + # Determine skill path for the skills array + # Extract language suffix from skill name (e.g., azure-cosmos-db-py -> py) + $languageMap = @{ + '-py' = 'azure-sdk-python' + '-dotnet' = 'azure-sdk-dotnet' + '-ts' = 'azure-sdk-typescript' + '-java' = 'azure-sdk-java' + '-rust' = 'azure-sdk-rust' + } + + $language = 'py' # default + foreach ($suffix in $languageMap.Keys) { + if ($skillName -match "$suffix`$") { + $language = $languageMap[$suffix] + break + } + } + + $skillPath = "../../../../.github/plugins/$language/skills/$skillName" + + # Paths for experiment files + $experimentFile = Join-Path $vallyDir "skill_effectiveness_experiment.yaml" + $evalFile = Join-Path $vallyDir "skill_effectiveness_eval.yaml" + $sourceEvalFile = Join-Path $vallyDir "eval.yaml" + + # Create experiment file + if ((Test-Path $experimentFile) -and -not $Force) { + Write-Verbose "Skipping (exists): $experimentFile" + $skipped++ + } + else { + try { + New-ExperimentFile -SkillName $skillName -SkillPath $skillPath -OutputPath $experimentFile -DryRun:$DryRun + $created++ + } + catch { + Write-Error "Failed to create experiment file for $skillName : $_" + $failed++ + } + } + + # Create eval file (copy from eval.yaml) + if ((Test-Path $evalFile) -and -not $Force) { + Write-Verbose "Skipping (exists): $evalFile" + } + else { + try { + if (Copy-EvalFile -SourcePath $sourceEvalFile -DestPath $evalFile -DryRun:$DryRun) { + if (-not $DryRun) { + $created++ + } + } + else { + Write-Warning "Source eval.yaml not found for $skillName" + } + } + catch { + Write-Error "Failed to copy eval file for $skillName : $_" + $failed++ + } + } +} + +Write-Information "" +Write-Information "Summary:" +Write-Information "--------" +Write-Information "Created: $created" +Write-Information "Skipped: $skipped" +if ($failed -gt 0) { + Write-Information "Failed: $failed" +} +if ($DryRun) { + Write-Information "" + Write-Information "DRY RUN - No files were actually created. Use -Force to create files." +} diff --git a/tests/harness/copilot-client.test.ts b/tests/harness/copilot-client.test.ts new file mode 100644 index 00000000..53100cdc --- /dev/null +++ b/tests/harness/copilot-client.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; + +import { + CopilotGenerationError, + SkillCopilotClient, + classifyCopilotError, +} from "./copilot-client.js"; + +describe("SkillCopilotClient.extractCode", () => { + it("keeps assignment line for multiline constructor call without fences", () => { + const client = new SkillCopilotClient(process.cwd(), true); + + const response = [ + "from azure.identity import DefaultAzureCredential", + "from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter", + "", + "exporter = AzureMonitorTraceExporter(", + " credential=DefaultAzureCredential(),", + ' storage_directory="/path/to/storage",', + " disable_offline_storage=False,", + ")", + ].join("\n"); + + const extracted = ( + client as unknown as { extractCode: (r: string) => string } + ).extractCode(response); + + expect(extracted).toContain("exporter = AzureMonitorTraceExporter("); + expect(extracted).toContain("disable_offline_storage=False,"); + expect(extracted.trim().endsWith(")")).toBe(true); + }); + + it("prefers fenced code blocks when present", () => { + const client = new SkillCopilotClient(process.cwd(), true); + + const response = [ + "Here is the implementation:", + "```python", + "x = 1", + "print(x)", + "```", + ].join("\n"); + + const extracted = ( + client as unknown as { extractCode: (r: string) => string } + ).extractCode(response); + + expect(extracted).toBe("x = 1\nprint(x)"); + }); +}); + +describe("classifyCopilotError", () => { + it("classifies timeout and marks retryable", () => { + const classified = classifyCopilotError( + new Error("Timeout after 120000ms waiting for session.idle"), + ); + + expect(classified.kind).toBe("timeout"); + expect(classified.retryable).toBe(true); + }); + + it("classifies auth and marks non-retryable", () => { + const classified = classifyCopilotError( + new Error( + "Authentication failed: Failed to fetch GitHub CLI user login (401): Bad credentials", + ), + ); + + expect(classified.kind).toBe("auth"); + expect(classified.retryable).toBe(false); + }); + + it("returns existing classified errors unchanged", () => { + const original = new CopilotGenerationError("transient", "429", true); + const classified = classifyCopilotError(original); + + expect(classified).toBe(original); + }); +}); diff --git a/tests/harness/copilot-client.ts b/tests/harness/copilot-client.ts index 1e05bb6a..9f9cdb50 100644 --- a/tests/harness/copilot-client.ts +++ b/tests/harness/copilot-client.ts @@ -16,6 +16,73 @@ import type { import { DEFAULT_GENERATION_CONFIG } from "./types.js"; import { CopilotClient as SDKCopilotClient } from "@github/copilot-sdk"; +export type CopilotGenerationErrorKind = + | "timeout" + | "auth" + | "transient" + | "fatal"; + +export class CopilotGenerationError extends Error { + readonly kind: CopilotGenerationErrorKind; + readonly retryable: boolean; + + constructor( + kind: CopilotGenerationErrorKind, + message: string, + retryable: boolean, + ) { + super(message); + this.name = "CopilotGenerationError"; + this.kind = kind; + this.retryable = retryable; + } +} + +export function classifyCopilotError(error: unknown): CopilotGenerationError { + if (error instanceof CopilotGenerationError) { + return error; + } + + const message = error instanceof Error ? error.message : String(error); + const normalized = message.toLowerCase(); + + if ( + normalized.includes("authentication failed") || + normalized.includes("bad credentials") || + normalized.includes("failed to fetch github cli user login") || + normalized.includes("401") + ) { + return new CopilotGenerationError("auth", message, false); + } + + if ( + normalized.includes("timeout") || + normalized.includes("session.idle") || + normalized.includes("timed out") + ) { + return new CopilotGenerationError("timeout", message, true); + } + + if ( + normalized.includes("429") || + normalized.includes("rate limit") || + normalized.includes("econnreset") || + normalized.includes("etimedout") || + normalized.includes("enotfound") || + normalized.includes("socket hang up") || + normalized.includes("temporarily unavailable") || + normalized.includes("service unavailable") + ) { + return new CopilotGenerationError("transient", message, true); + } + + return new CopilotGenerationError("fatal", message, false); +} + +function sleep(ms: number): Promise { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + // ============================================================================= // Mock Client // ============================================================================= @@ -85,6 +152,8 @@ export class MockCopilotClient implements CopilotClient { export class SkillCopilotClient implements CopilotClient { private static readonly SKILLS_DIR = ".github/skills"; private static readonly PLUGINS_DIR = ".github/plugins"; + private static readonly DEFAULT_TIMEOUT_MS = 120000; + private static readonly DEFAULT_MAX_RETRIES = 2; private basePath: string; private skillsDir: string; @@ -227,23 +296,28 @@ Generate only code. Follow the patterns from the skill documentation exactly. ): Promise { const startTime = Date.now(); const fullPrompt = this.buildPrompt(prompt, skillContext); + const timeoutMs = this.getTimeoutMs(); + const maxRetries = this.getMaxRetries(); - const client = new SDKCopilotClient(); - - let rawResponse = ""; - - try { - const session = await client.createSession({ - model: config.model, - }); + for (let attempt = 0; attempt <= maxRetries; attempt++) { + const client = new SDKCopilotClient(); + let session: unknown; try { - const response = await session.sendAndWait( - { prompt: fullPrompt }, - 120000, - ); - - rawResponse = response?.data?.content ?? ""; + session = await client.createSession({ + model: config.model, + }); + + const response = await ( + session as { + sendAndWait: ( + request: { prompt: string }, + timeout: number, + ) => Promise<{ data?: { content?: string } }>; + } + ).sendAndWait({ prompt: fullPrompt }, timeoutMs); + + const rawResponse = response?.data?.content ?? ""; const code = this.extractCode(rawResponse); return { @@ -255,20 +329,51 @@ Generate only code. Follow the patterns from the skill documentation exactly. durationMs: Date.now() - startTime, rawResponse, }; + } catch (error) { + const classified = classifyCopilotError(error); + if (!classified.retryable || attempt === maxRetries) { + throw classified; + } + + // Exponential backoff with jitter for transient SDK/network failures. + const backoffMs = Math.min(1000 * 2 ** attempt, 8000); + const jitterMs = Math.floor(Math.random() * 300); + await sleep(backoffMs + jitterMs); } finally { - const s = session as unknown as { + const s = session as { disconnect?: () => Promise; destroy?: () => Promise; }; - if (typeof s.disconnect === "function") { + if (s && typeof s.disconnect === "function") { await s.disconnect(); - } else if (typeof s.destroy === "function") { + } else if (s && typeof s.destroy === "function") { await s.destroy(); } + await client.stop(); } - } finally { - await client.stop(); } + + throw new CopilotGenerationError( + "fatal", + "Code generation failed after all retry attempts.", + false, + ); + } + + private getTimeoutMs(): number { + const raw = process.env["HARNESS_COPILOT_TIMEOUT_MS"]; + const parsed = raw ? Number.parseInt(raw, 10) : NaN; + return Number.isFinite(parsed) && parsed > 0 + ? parsed + : SkillCopilotClient.DEFAULT_TIMEOUT_MS; + } + + private getMaxRetries(): number { + const raw = process.env["HARNESS_COPILOT_MAX_RETRIES"]; + const parsed = raw ? Number.parseInt(raw, 10) : NaN; + return Number.isFinite(parsed) && parsed >= 0 + ? parsed + : SkillCopilotClient.DEFAULT_MAX_RETRIES; } /** @@ -295,9 +400,12 @@ Generate only code. Follow the patterns from the skill documentation exactly. const codeLines: string[] = []; let inCode = false; + const assignmentRegex = /^[A-Za-z_][A-Za-z0-9_]*\s*=\s*.+$/; + const callStartRegex = /^[A-Za-z_][A-Za-z0-9_.]*\s*\(.+$/; + for (const line of lines) { - // Heuristic: lines starting with import, def, class, or indented - if ( + const trimmed = line.trim(); + const isCodeLikeLine = line.startsWith("import ") || line.startsWith("from ") || line.startsWith("def ") || @@ -307,8 +415,12 @@ Generate only code. Follow the patterns from the skill documentation exactly. line.startsWith("let ") || line.startsWith("using ") || line.startsWith(" ") || - line.startsWith("\t") - ) { + line.startsWith("\t") || + assignmentRegex.test(trimmed) || + callStartRegex.test(trimmed); + + // Heuristic: lines starting with import, def, class, or indented + if (isCodeLikeLine) { inCode = true; codeLines.push(line); } else if (inCode && line.trim() === "") { diff --git a/tests/harness/ralph-loop.test.ts b/tests/harness/ralph-loop.test.ts index c322be6e..019c1b42 100644 --- a/tests/harness/ralph-loop.test.ts +++ b/tests/harness/ralph-loop.test.ts @@ -55,10 +55,11 @@ class MockClient implements CopilotClient { _prompt: string, _skillName: string, _config?: GenerationConfig, - _scenarioName?: string + _scenarioName?: string, ): Promise { this.callCount++; - const code = this.responses.get(this.callCount) ?? "# Default mock response\npass"; + const code = + this.responses.get(this.callCount) ?? "# Default mock response\npass"; return { code, prompt: _prompt, @@ -111,13 +112,15 @@ class MockEvaluator { skillName: "test-skill", scenario, generatedCode: code, + rawResponse: code, findings, matchedCorrect: [], matchedIncorrect: score < 70 ? ["some-section"] : [], score, passed: score >= 50, errorCount: findings.filter((f) => f.severity === Severity.ERROR).length, - warningCount: findings.filter((f) => f.severity === Severity.WARNING).length, + warningCount: findings.filter((f) => f.severity === Severity.WARNING) + .length, }; } } @@ -161,7 +164,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80 } + { qualityThreshold: 80 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -184,7 +187,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 10 } + { qualityThreshold: 80, improvementThreshold: 10 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -197,18 +200,14 @@ describe("RalphLoopController", () => { }); it("should stop at max iterations if threshold not met", async () => { - mockClient.setResponses([ - "# Attempt 1", - "# Attempt 2", - "# Attempt 3", - ]); + mockClient.setResponses(["# Attempt 1", "# Attempt 2", "# Attempt 3"]); mockEvaluator.setScores([30, 40, 50]); // Never reaches 80 const controller = new RalphLoopController( criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 5 } + { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 5 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -229,7 +228,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { earlyStopOnPerfect: true } + { earlyStopOnPerfect: true }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -247,7 +246,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 5 } + { qualityThreshold: 80, improvementThreshold: 5 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -265,7 +264,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 5 } + { qualityThreshold: 80, improvementThreshold: 5 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -285,7 +284,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 0 } + { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 0 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -305,7 +304,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 10 } + { qualityThreshold: 80, improvementThreshold: 10 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -334,7 +333,11 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, includeFeedback: true, improvementThreshold: 10 } + { + qualityThreshold: 80, + includeFeedback: true, + improvementThreshold: 10, + }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -353,7 +356,7 @@ describe("RalphLoopController", () => { const controller = new RalphLoopController( criteria, mockEvaluator as unknown as CodeEvaluator, - mockClient + mockClient, ); const result = await controller.run("Generate code", "test-scenario"); diff --git a/tests/harness/runner.ts b/tests/harness/runner.ts index f106cea9..704ac972 100644 --- a/tests/harness/runner.ts +++ b/tests/harness/runner.ts @@ -20,8 +20,17 @@ import type { GenerationConfig, Finding, } from "./types.js"; -import { DEFAULT_GENERATION_CONFIG, Severity, createFinding } from "./types.js"; -import { SkillCopilotClient, checkCopilotAvailable } from "./copilot-client.js"; +import { + DEFAULT_GENERATION_CONFIG, + Severity, + createEvaluationResult, + createFinding, +} from "./types.js"; +import { + CopilotGenerationError, + SkillCopilotClient, + checkCopilotAvailable, +} from "./copilot-client.js"; import { AcceptanceCriteriaLoader } from "./criteria-loader.js"; import { CodeEvaluator } from "./evaluator.js"; import { @@ -282,19 +291,41 @@ export class SkillEvaluationRunner { ); } - // Generate code - const genResult = await this.copilotClient.generate( - scenario.prompt, - skillName, - suite.config, - scenario.name, - ); + let evalResult: EvaluationResult; - // Evaluate - const evalResult = evaluator.evaluate(genResult.code, scenario.name); + try { + // Generate code + const genResult = await this.copilotClient.generate( + scenario.prompt, + skillName, + suite.config, + scenario.name, + ); - // Add scenario-specific checks - this.checkScenarioPatterns(evalResult, scenario, genResult.code); + // Evaluate + evalResult = evaluator.evaluate(genResult.code, scenario.name); + evalResult.rawResponse = genResult.rawResponse; + + // Add scenario-specific checks + this.checkScenarioPatterns(evalResult, scenario, genResult.code); + } catch (error) { + const kind = + error instanceof CopilotGenerationError ? error.kind : "fatal"; + const message = error instanceof Error ? error.message : String(error); + + evalResult = createEvaluationResult(skillName, scenario.name, ""); + evalResult.passed = false; + evalResult.errorCount = 1; + evalResult.score = 0; + evalResult.findings.push( + createFinding({ + severity: Severity.ERROR, + rule: `runtime:${kind}`, + message: `Code generation failed (${kind}): ${message}`, + suggestion: this.getGenerationFailureSuggestion(kind), + }), + ); + } results.push(evalResult); @@ -394,6 +425,21 @@ export class SkillEvaluationRunner { } } + private getGenerationFailureSuggestion( + kind: "timeout" | "auth" | "transient" | "fatal", + ): string { + switch (kind) { + case "timeout": + return "Retry with HARNESS_COPILOT_TIMEOUT_MS set higher (for example 240000), or re-run this scenario."; + case "auth": + return "Refresh GitHub authentication (gh auth login) or set GH_TOKEN/GITHUB_TOKEN with valid Copilot access."; + case "transient": + return "This looks transient; retry the run. If it repeats, verify network stability and API availability."; + default: + return "Inspect diagnostics and rerun with --verbose for additional context."; + } + } + private printFinding(finding: Finding): void { const severityStyle = this.getSeverityStyle(finding.severity); const severityLabel = finding.severity.toUpperCase(); @@ -621,6 +667,7 @@ function summaryToDict(summary: EvaluationSummary): Record { skill_name: r.skillName, scenario: r.scenario, generated_code: r.generatedCode, + raw_response: r.rawResponse, passed: r.passed, score: r.score, error_count: r.errorCount, @@ -770,6 +817,7 @@ function convertRalphToSummary(ralph: RalphLoopSummary): EvaluationSummary { skillName: ralph.skillName, scenario: scenarioName, generatedCode: lastIteration.generatedCode, + rawResponse: "", findings: lastIteration.findings, matchedCorrect: [], matchedIncorrect: [], @@ -813,6 +861,8 @@ interface CLIOptions { threshold?: number; } +type CliErrorKind = "timeout" | "auth" | "transient" | "fatal"; + interface AllSkillsSummary { totalSkills: number; passedSkills: number; @@ -826,6 +876,52 @@ interface AllSkillsSummary { skills: EvaluationSummary[]; } +function classifyCliError(error: unknown): CliErrorKind { + if (error instanceof CopilotGenerationError) { + return error.kind; + } + + const message = error instanceof Error ? error.message.toLowerCase() : ""; + if ( + message.includes("authentication failed") || + message.includes("bad credentials") || + message.includes("401") + ) { + return "auth"; + } + if (message.includes("timeout") || message.includes("session.idle")) { + return "timeout"; + } + if (message.includes("rate limit") || message.includes("429")) { + return "transient"; + } + return "fatal"; +} + +function writeCliErrorOutput( + options: CLIOptions, + skillName: string | undefined, + error: unknown, +): void { + const message = error instanceof Error ? error.message : String(error); + const payload = { + status: "ERROR", + error_kind: classifyCliError(error), + error: message, + skill: skillName ?? null, + timestamp: new Date().toISOString(), + }; + const output = JSON.stringify(payload, null, 2); + + if (options.outputFile) { + writeFileSync(options.outputFile, output); + console.log(`Results written to: ${options.outputFile}`); + return; + } + + console.log(output); +} + async function main(): Promise { const program = new Command(); @@ -1087,6 +1183,10 @@ async function main(): Promise { } } catch (err) { const message = err instanceof Error ? err.message : String(err); + if (options.output === "json") { + writeCliErrorOutput(options, skillArg, err); + return 1; + } console.log(chalk.red(`Error: ${message}`)); return 1; } diff --git a/tests/harness/types.ts b/tests/harness/types.ts index 715fb494..37e5b525 100644 --- a/tests/harness/types.ts +++ b/tests/harness/types.ts @@ -83,6 +83,7 @@ export interface EvaluationResult { skillName: string; scenario: string; generatedCode: string; + rawResponse: string; findings: Finding[]; matchedCorrect: string[]; matchedIncorrect: string[]; @@ -110,7 +111,7 @@ export interface GenerationConfig { * Default generation configuration. */ export const DEFAULT_GENERATION_CONFIG: GenerationConfig = { - model: "gpt-4", + model: "gpt-5.5", maxTokens: 2000, temperature: 0.3, includeSkillContext: true, @@ -175,7 +176,7 @@ export interface CopilotClient { prompt: string, skillName: string, config?: GenerationConfig, - scenarioName?: string + scenarioName?: string, ): Promise; } @@ -223,12 +224,13 @@ export function detectLanguage(skillName: string): Language { export function createEvaluationResult( skillName: string, scenario: string, - generatedCode: string + generatedCode: string, ): EvaluationResult { return { skillName, scenario, generatedCode, + rawResponse: "", findings: [], matchedCorrect: [], matchedIncorrect: [], @@ -242,7 +244,9 @@ export function createEvaluationResult( /** * Create an empty code pattern. */ -export function createCodePattern(partial: Partial = {}): CodePattern { +export function createCodePattern( + partial: Partial = {}, +): CodePattern { return { code: "", language: "python", @@ -256,7 +260,9 @@ export function createCodePattern(partial: Partial = {}): CodePatte /** * Create an empty validation rule. */ -export function createValidationRule(partial: Partial = {}): ValidationRule { +export function createValidationRule( + partial: Partial = {}, +): ValidationRule { return { name: "", description: "", @@ -274,7 +280,7 @@ export function createValidationRule(partial: Partial = {}): Val * Create an empty acceptance criteria. */ export function createAcceptanceCriteria( - partial: Partial = {} + partial: Partial = {}, ): AcceptanceCriteria { return { skillName: "", diff --git a/tests/package-lock.json b/tests/package-lock.json index 1c56a6b4..ee981860 100644 --- a/tests/package-lock.json +++ b/tests/package-lock.json @@ -8,23 +8,120 @@ "name": "agent-skills-tests", "version": "1.0.0", "dependencies": { - "@github/copilot-sdk": "^0.1.20", + "@github/copilot-sdk": "^1.0.3", "chalk": "^5.3.0", "commander": "^12.1.0", "yaml": "^2.4.5", "zod": "^3.23.8" }, "devDependencies": { + "@microsoft/vally": "^0.7.0", "@types/node": "^20.14.0", + "@vitest/coverage-v8": "^4.1.9", "eslint": "^9.5.0", "tsx": "^4.15.6", "typescript": "^5.4.5", - "vitest": "^1.6.0" + "vite": "^8.1.0", + "vitest": "^4.1.9" }, "engines": { "node": ">=20.0.0" } }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", @@ -612,26 +709,31 @@ } }, "node_modules/@github/copilot": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-0.0.411.tgz", - "integrity": "sha512-I3/7gw40Iu1O+kTyNPKJHNqDRyOebjsUW6wJsvSVrOpT0TNa3/lfm8xdS2XUuJWkp+PgEG/PRwF7u3DVNdP7bQ==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.65.tgz", + "integrity": "sha512-J1XvLuOiVpiAi/E1MBICBymszCgdGLnZxokosXzGcmcjEVZd+QSDoW/kPRHq6oEyBT9SDASPcjCEZ9Q0rLJllg==", "license": "SEE LICENSE IN LICENSE.md", + "dependencies": { + "detect-libc": "^2.1.2" + }, "bin": { "copilot": "npm-loader.js" }, "optionalDependencies": { - "@github/copilot-darwin-arm64": "0.0.411", - "@github/copilot-darwin-x64": "0.0.411", - "@github/copilot-linux-arm64": "0.0.411", - "@github/copilot-linux-x64": "0.0.411", - "@github/copilot-win32-arm64": "0.0.411", - "@github/copilot-win32-x64": "0.0.411" + "@github/copilot-darwin-arm64": "1.0.65", + "@github/copilot-darwin-x64": "1.0.65", + "@github/copilot-linux-arm64": "1.0.65", + "@github/copilot-linux-x64": "1.0.65", + "@github/copilot-linuxmusl-arm64": "1.0.65", + "@github/copilot-linuxmusl-x64": "1.0.65", + "@github/copilot-win32-arm64": "1.0.65", + "@github/copilot-win32-x64": "1.0.65" } }, "node_modules/@github/copilot-darwin-arm64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-0.0.411.tgz", - "integrity": "sha512-dtr+iHxTS4f8HlV2JT9Fp0FFoxuiPWCnU3XGmrHK+rY6bX5okPC2daU5idvs77WKUGcH8yHTZtfbKYUiMxKosw==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-1.0.65.tgz", + "integrity": "sha512-NFc4xIstZNiIuAYkurQT5DVtbZjBoZ/z6yt/Ffcom7Y5QGjfpN4BFuekv9k+OADRioxxR99NgmhjbuNPWtQhNQ==", "cpu": [ "arm64" ], @@ -645,9 +747,9 @@ } }, "node_modules/@github/copilot-darwin-x64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-0.0.411.tgz", - "integrity": "sha512-zhdbQCbPi1L4iHClackSLx8POfklA+NX9RQLuS48HlKi/0KI/JlaDA/bdbIeMR79wjif5t9gnc/m+RTVmHlRtA==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-1.0.65.tgz", + "integrity": "sha512-0wtV22KmTa12VbqWRRkgvJcBz/oIbszfcIpyDWGc4MzbCVksajQ3TWVQ6c7Sdzj5RifCaYdkHAX2zuIYXYlLoQ==", "cpu": [ "x64" ], @@ -661,12 +763,15 @@ } }, "node_modules/@github/copilot-linux-arm64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-0.0.411.tgz", - "integrity": "sha512-oZYZ7oX/7O+jzdTUcHkfD1A8YnNRW6mlUgdPjUg+5rXC43bwIdyatAnc0ObY21m9h8ghxGqholoLhm5WnGv1LQ==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-1.0.65.tgz", + "integrity": "sha512-dOwdy/YbTXQN/+x2v4ZgiDycdRtWElyHxPuA6ail3yJDt0nagwn8OYAA/diBLPMAJuuBXiOZGvvb9fGRuh7Xgg==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -677,12 +782,15 @@ } }, "node_modules/@github/copilot-linux-x64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-0.0.411.tgz", - "integrity": "sha512-nnXrKANmmGnkwa3ROlKdAhVNOx8daeMSE8Xh0o3ybKckFv4s38blhKdcxs0RJQRxgAk4p7XXGlDDKNRhurqF1g==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-1.0.65.tgz", + "integrity": "sha512-al/1a/l/GrpHtygTxt7PZspmv0eHBPdAZ5B31J7Hv/GRdVZM4STCC9dCIOSUFsOX2fhaKD8yLfz4HureSYs23g==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -692,18 +800,56 @@ "copilot-linux-x64": "copilot" } }, + "node_modules/@github/copilot-linuxmusl-arm64": { + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-arm64/-/copilot-linuxmusl-arm64-1.0.65.tgz", + "integrity": "sha512-xccQeJSR45xyoaL7J5mZjtU++dmte+ZCDQkIlrpTn2yuMl2LWriBvorQ1P2MwVnXmIiW/GHi93B+lNtsybA9yw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-x64": { + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-x64/-/copilot-linuxmusl-x64-1.0.65.tgz", + "integrity": "sha512-RHPVUaqjSrhKHQ2EpfGKWErnV+R5elGIZaHXPKO10zpSaQD9b/C9u6nLigZnBuT/8sCaJpVrazPMwOYvYA62aw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-x64": "copilot" + } + }, "node_modules/@github/copilot-sdk": { - "version": "0.1.25", - "resolved": "https://registry.npmjs.org/@github/copilot-sdk/-/copilot-sdk-0.1.25.tgz", - "integrity": "sha512-hIgYLPXzWw9bNgrsD5BLKmgVH20ow5Or5UyVXfVe3YgeiaTgFxC4jWSAVHLGB6ufHZUrvbjppcq2dWK63FmDRA==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@github/copilot-sdk/-/copilot-sdk-1.0.4.tgz", + "integrity": "sha512-c6/gpatP7LAuP9stlc2uXXOrB+TJMFSs3Jrzu9FG8lJJYFGmdzbBvGz2UdL1jww84fp6D7cohEZa4UGq1+iuyA==", "license": "MIT", "dependencies": { - "@github/copilot": "^0.0.411", + "@github/copilot": "^1.0.65", "vscode-jsonrpc": "^8.2.1", "zod": "^4.3.6" }, "engines": { - "node": ">=20.0.0" + "node": "^20.19.0 || >=22.12.0" } }, "node_modules/@github/copilot-sdk/node_modules/zod": { @@ -716,9 +862,9 @@ } }, "node_modules/@github/copilot-win32-arm64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-0.0.411.tgz", - "integrity": "sha512-h+Bovb2YVCQSeELZOO7zxv8uht45XHcvAkFbRsc1gf9dl109sSUJIcB4KAhs8Aznk28qksxz7kvdSgUWyQBlIA==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-1.0.65.tgz", + "integrity": "sha512-/vSE/t9Wm3eFSWpxlKyn/oL8OAVOB0yFO7ECxhgbtiqNrBd1tgpYh1k7IXBIWa/saxlV1+de6DEmPuQfx3Z0bg==", "cpu": [ "arm64" ], @@ -732,9 +878,9 @@ } }, "node_modules/@github/copilot-win32-x64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-0.0.411.tgz", - "integrity": "sha512-xmOgi1lGvUBHQJWmq5AK1EP95+Y8xR4TFoK9OCSOaGbQ+LFcX2jF7iavnMolfWwddabew/AMQjsEHlXvbgMG8Q==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-1.0.65.tgz", + "integrity": "sha512-wjVWXepET+SpFg8z8V43ZiTy6X1OerCb7yu3QZKNNJ3zY9z20goihPXQCDWkiJpGzszNSgfrsiqUzpUsC9qS0A==", "cpu": [ "x64" ], @@ -799,17 +945,14 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@jest/schemas": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", - "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", "dev": true, "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.27.8" - }, "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + "node": ">=6.0.0" } }, "node_modules/@jridgewell/sourcemap-codec": { @@ -819,24 +962,84 @@ "dev": true, "license": "MIT" }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz", - "integrity": "sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==", - "cpu": [ - "arm" - ], + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@microsoft/vally": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@microsoft/vally/-/vally-0.7.0.tgz", + "integrity": "sha512-GYxSgub10SC6X1d/vVUCOorGzAMWIUN7uk2zq65pPle0CfKdWy+HiEpnJkgrXGat7kZxzx+jM9lrffW3nfKpmw==", + "dev": true, + "dependencies": { + "@github/copilot-sdk": "^1.0.3", + "@opentelemetry/api": "^1.9.1", + "js-tiktoken": "^1.0.21", + "picomatch": "^4.0.4", + "yaml": "^2.9.0", + "zod": "^4.4.3" + } + }, + "node_modules/@microsoft/vally/node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", "dev": true, "license": "MIT", "optional": true, - "os": [ - "android" - ] + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.57.1.tgz", - "integrity": "sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==", + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.137.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.137.0.tgz", + "integrity": "sha512-WT+Gb24i8hmvo85AIv2oEYouEXkRlKAlT9WaCa3TfLgNCN+GhrJOGZuIlMouAh38Qe4QOx26eUOVsq70qXrywA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.3.tgz", + "integrity": "sha512-DT6Z3PhvioeHMvxo+xHc3KtqggrI7CCTXCmC2h/5zUlp5jVitv7XEy+9q5/7v8IolhlioawpMo8Kg0EEBy7J0g==", "cpu": [ "arm64" ], @@ -845,12 +1048,15 @@ "optional": true, "os": [ "android" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.57.1.tgz", - "integrity": "sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==", + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.3.tgz", + "integrity": "sha512-0NwgwsjM7LrsuVnXMK3koTpagBNOhloc/BNjKqZjv4V5zI5r13qx69uVhRx+o5Z0yy4Hzq+lpy7TAgUG/ocvrw==", "cpu": [ "arm64" ], @@ -859,12 +1065,15 @@ "optional": true, "os": [ "darwin" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.57.1.tgz", - "integrity": "sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==", + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.3.tgz", + "integrity": "sha512-YtiBp4disu6V560loT6PjMdiRaWmVvDNrUunAalbiFx2ggeJwxdAsgZMcoGP17uyAsTwAj5V1niksxlHnVQ1Sw==", "cpu": [ "x64" ], @@ -873,26 +1082,15 @@ "optional": true, "os": [ "darwin" - ] - }, - "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.57.1.tgz", - "integrity": "sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.57.1.tgz", - "integrity": "sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==", + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.3.tgz", + "integrity": "sha512-yD3EkEdXk2LypPxnf/kSZHirarsI8gcPzc62SukhR9VJTyvV+F9Q/GxWNuCojc7sXyuVC4DxRGhdDK4X8VSsbw==", "cpu": [ "x64" ], @@ -901,26 +1099,15 @@ "optional": true, "os": [ "freebsd" - ] - }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.57.1.tgz", - "integrity": "sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==", - "cpu": [ - "arm" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.57.1.tgz", - "integrity": "sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==", + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.3.tgz", + "integrity": "sha512-c+8vieQbsD7HNAHKIA34w0GJ9FedFFuJGD+7E6vz7Q3uqAIugL5p45fhlsj4UaAsHpcmlqugBWMhA0/j7o0sIg==", "cpu": [ "arm" ], @@ -929,180 +1116,135 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.57.1.tgz", - "integrity": "sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.57.1.tgz", - "integrity": "sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==", + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.3.tgz", + "integrity": "sha512-50jD0uUwLvur7Zz9LHz17kaAdTPjn5wN93hEgjvmYFRZwiR7ZJYovTd5ipyWJDAnXKvZ+wgc+/Ika6dwSF5OcA==", "cpu": [ "arm64" ], "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.57.1.tgz", - "integrity": "sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==", - "cpu": [ - "loong64" + "libc": [ + "glibc" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.57.1.tgz", - "integrity": "sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==", - "cpu": [ - "loong64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.57.1.tgz", - "integrity": "sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==", + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.3.tgz", + "integrity": "sha512-BO9+oPL8K9poZJBfYPsXNtYjPE5uM3qeehT3aFcW4LITOl+iSqhp0abzjR2nWBUNjIZeKXjAEWBZ64WjNoHd6w==", "cpu": [ - "ppc64" + "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.57.1.tgz", - "integrity": "sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==", + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.3.tgz", + "integrity": "sha512-f3VpLB1vQ0Eo6ecr/6cekLnvYMFF4YBFoVGkfkvPLq1bAkbAwHYQPZKoAmG6OJyTcxxoC+AvezGx/S1obNC0Mw==", "cpu": [ "ppc64" ], "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.57.1.tgz", - "integrity": "sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==", - "cpu": [ - "riscv64" + "libc": [ + "glibc" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.57.1.tgz", - "integrity": "sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==", - "cpu": [ - "riscv64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.57.1.tgz", - "integrity": "sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==", + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.3.tgz", + "integrity": "sha512-AmurZ26Pqx/RI9N1gzEOCklkKXl927yjfXWUUS0O7Puh8ARM/Ob8qfrD3qnWksScdw6cSrW5PSHE9DyLu7+PtA==", "cpu": [ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.57.1.tgz", - "integrity": "sha512-ABca4ceT4N+Tv/GtotnWAeXZUZuM/9AQyCyKYyKnpk4yoA7QIAuBt6Hkgpw8kActYlew2mvckXkvx0FfoInnLg==", + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.3.tgz", + "integrity": "sha512-JJpqs8bRGITDOdbkNKnlojzBabbOHrqjSvDr0IVsZObE1lBcPjxItUEY9eWIDbxaJ3cGrXPWGfGkIxFijg/URg==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.57.1.tgz", - "integrity": "sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==", + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.3.tgz", + "integrity": "sha512-rSJcdjPxzA/by/6/rYs+v+bXU7UjvnbUWz8MJb6kh6+knqB1dCrtHg0uu7C/4haqJvqdkYHQ5IGn+tCH9GLW/g==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.57.1.tgz", - "integrity": "sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==", - "cpu": [ - "x64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.57.1.tgz", - "integrity": "sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==", + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.3.tgz", + "integrity": "sha512-hQ3/PYkDJICgevvyNcVrihVeqq7k1Pp3VZ9lY+dauAYUJKO+auqApvANhvR1An9BhmqYKvW2Mu1F9u4DXSMLxQ==", "cpu": [ "arm64" ], @@ -1111,54 +1253,51 @@ "optional": true, "os": [ "openharmony" - ] - }, - "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.57.1.tgz", - "integrity": "sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.57.1.tgz", - "integrity": "sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==", + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.3.tgz", + "integrity": "sha512-Elcv/BtML9lXrV6JuKITc/grN2kYV9gjsQpW8Jfw4ioK0TOkjBjye0nnyqQNy9STNaI20lXNaQBRrD5gSgR0Yg==", "cpu": [ - "ia32" + "wasm32" ], "dev": true, "license": "MIT", "optional": true, - "os": [ - "win32" - ] + "dependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.57.1.tgz", - "integrity": "sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==", + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.3.tgz", + "integrity": "sha512-2DrEfhluH9yhiaFApmsjsjwrSYbNcY1oFTzYSP1a535jDbV98zCFanA/96TBUd0iDFcxGmw9QRExwGCXz3U+/g==", "cpu": [ - "x64" + "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.57.1.tgz", - "integrity": "sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==", + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.3.tgz", + "integrity": "sha512-OL4OMk7UPXOeVGGd3qo5zJyPIljf4AFgk5QAkPPS+OoLuOOozhuaQGC18MxVTnw/06q93gShAJzlwnSCY9YtqA==", "cpu": [ "x64" ], @@ -1167,29 +1306,68 @@ "optional": true, "os": [ "win32" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@sinclair/typebox": { - "version": "0.27.10", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.10.tgz", - "integrity": "sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==", + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", "dev": true, "license": "MIT" }, - "node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", "dev": true, "license": "MIT" }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", "dev": true, - "license": "MIT" - }, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "20.19.33", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.33.tgz", @@ -1200,104 +1378,145 @@ "undici-types": "~6.21.0" } }, - "node_modules/@vitest/expect": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-1.6.1.tgz", - "integrity": "sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog==", + "node_modules/@vitest/coverage-v8": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.9.tgz", + "integrity": "sha512-G9/lgqibheLVBDRuya45EbsEXTYcWoSG+TLg7i2axuzx0Eq62eXn+aWXyaVdV5vKvFSWd6ywcX8hA7la9Pvu8g==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "1.6.1", - "@vitest/utils": "1.6.1", - "chai": "^4.3.10" + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.1.9", + "ast-v8-to-istanbul": "^1.0.0", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.2", + "obug": "^2.1.1", + "std-env": "^4.0.0-rc.1", + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.1.9", + "vitest": "4.1.9" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } } }, - "node_modules/@vitest/runner": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-1.6.1.tgz", - "integrity": "sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA==", + "node_modules/@vitest/expect": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.9.tgz", + "integrity": "sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "1.6.1", - "p-limit": "^5.0.0", - "pathe": "^1.1.1" + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/runner/node_modules/p-limit": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", - "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", + "node_modules/@vitest/mocker": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.9.tgz", + "integrity": "sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==", "dev": true, "license": "MIT", "dependencies": { - "yocto-queue": "^1.0.0" + "@vitest/spy": "4.1.9", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" }, - "engines": { - "node": ">=18" + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.9.tgz", + "integrity": "sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/runner/node_modules/yocto-queue": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", - "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "node_modules/@vitest/runner": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.9.tgz", + "integrity": "sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==", "dev": true, "license": "MIT", - "engines": { - "node": ">=12.20" + "dependencies": { + "@vitest/utils": "4.1.9", + "pathe": "^2.0.3" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/snapshot": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.6.1.tgz", - "integrity": "sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ==", + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.9.tgz", + "integrity": "sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==", "dev": true, "license": "MIT", "dependencies": { - "magic-string": "^0.30.5", - "pathe": "^1.1.1", - "pretty-format": "^29.7.0" + "@vitest/pretty-format": "4.1.9", + "@vitest/utils": "4.1.9", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/spy": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-1.6.1.tgz", - "integrity": "sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw==", + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.9.tgz", + "integrity": "sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==", "dev": true, "license": "MIT", - "dependencies": { - "tinyspy": "^2.2.0" - }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/utils": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-1.6.1.tgz", - "integrity": "sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g==", + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.9.tgz", + "integrity": "sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==", "dev": true, "license": "MIT", "dependencies": { - "diff-sequences": "^29.6.3", - "estree-walker": "^3.0.3", - "loupe": "^2.3.7", - "pretty-format": "^29.7.0" + "@vitest/pretty-format": "4.1.9", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" @@ -1326,19 +1545,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/acorn-walk": { - "version": "8.3.5", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", - "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.11.0" - }, - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -1356,19 +1562,6 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -1377,13 +1570,25 @@ "license": "Python-2.0" }, "node_modules/assertion-error": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", - "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", "dev": true, "license": "MIT", "engines": { - "node": "*" + "node": ">=12" + } + }, + "node_modules/ast-v8-to-istanbul": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.4.tgz", + "integrity": "sha512-0bC0/4bTSrnwdhU3IsZDwEdojvuPrSg59OYZfKsLRtJZ0u8VBx9DebfqqG8bRdCC0I7vjgxmPi41P0lpkhJHtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" } }, "node_modules/balanced-match": { @@ -1393,6 +1598,27 @@ "dev": true, "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -1404,16 +1630,6 @@ "concat-map": "0.0.1" } }, - "node_modules/cac": { - "version": "6.7.14", - "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", - "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1425,22 +1641,13 @@ } }, "node_modules/chai": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", - "integrity": "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", - "dependencies": { - "assertion-error": "^1.1.0", - "check-error": "^1.0.3", - "deep-eql": "^4.1.3", - "get-func-name": "^2.0.2", - "loupe": "^2.3.6", - "pathval": "^1.1.1", - "type-detect": "^4.1.0" - }, "engines": { - "node": ">=4" + "node": ">=18" } }, "node_modules/chalk": { @@ -1455,19 +1662,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/check-error": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.3.tgz", - "integrity": "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==", - "dev": true, - "license": "MIT", - "dependencies": { - "get-func-name": "^2.0.2" - }, - "engines": { - "node": "*" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1504,10 +1698,10 @@ "dev": true, "license": "MIT" }, - "node_modules/confbox": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", - "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "dev": true, "license": "MIT" }, @@ -1544,19 +1738,6 @@ } } }, - "node_modules/deep-eql": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.4.tgz", - "integrity": "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==", - "dev": true, - "license": "MIT", - "dependencies": { - "type-detect": "^4.0.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -1564,16 +1745,22 @@ "dev": true, "license": "MIT" }, - "node_modules/diff-sequences": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", - "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", - "dev": true, - "license": "MIT", + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + "node": ">=8" } }, + "node_modules/es-module-lexer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.2.0.tgz", + "integrity": "sha512-3lGxdTXCLfe1MYfTz1y2ksAAUM4NAOP6rPEjxGJVKO7TZ5+tvHCaQWGpC4Y3IXvW3ece0Cz1cIP4FWBxOnGCTQ==", + "dev": true, + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", @@ -1826,28 +2013,14 @@ "node": ">=0.10.0" } }, - "node_modules/execa": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", - "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", "dev": true, - "license": "MIT", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^8.0.1", - "human-signals": "^5.0.0", - "is-stream": "^3.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^5.1.0", - "onetime": "^6.0.0", - "signal-exit": "^4.1.0", - "strip-final-newline": "^3.0.0" - }, + "license": "Apache-2.0", "engines": { - "node": ">=16.17" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" + "node": ">=12.0.0" } }, "node_modules/fast-deep-equal": { @@ -1871,6 +2044,24 @@ "dev": true, "license": "MIT" }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -1937,29 +2128,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/get-func-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", - "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/get-stream": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", - "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-tsconfig": { "version": "4.13.6", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.6.tgz", @@ -2009,15 +2177,12 @@ "node": ">=8" } }, - "node_modules/human-signals": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", - "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=16.17.0" - } + "license": "MIT" }, "node_modules/ignore": { "version": "5.3.2", @@ -2079,19 +2244,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", - "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -2099,10 +2251,59 @@ "dev": true, "license": "ISC" }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "dev": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", - "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", "dev": true, "license": "MIT" }, @@ -2164,56 +2365,302 @@ "node": ">= 0.8.0" } }, - "node_modules/local-pkg": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-0.5.1.tgz", - "integrity": "sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ==", + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", "dev": true, - "license": "MIT", + "license": "MPL-2.0", "dependencies": { - "mlly": "^1.7.3", - "pkg-types": "^1.2.1" + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], "engines": { - "node": ">=14" + "node": ">= 12.0.0" }, "funding": { - "url": "https://github.com/sponsors/antfu" + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^5.0.0" - }, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], "engines": { - "node": ">=10" + "node": ">= 12.0.0" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], "dev": true, - "license": "MIT" + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } }, - "node_modules/loupe": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.7.tgz", - "integrity": "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==", + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, "license": "MIT", "dependencies": { - "get-func-name": "^2.0.1" + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -2224,21 +2671,29 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "node_modules/magicast": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.3.tgz", + "integrity": "sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==", "dev": true, - "license": "MIT" + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.3", + "@babel/types": "^7.29.0", + "source-map-js": "^1.2.1" + } }, - "node_modules/mimic-fn": { + "node_modules/make-dir": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", - "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", "dev": true, "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, "engines": { - "node": ">=12" + "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2257,26 +2712,6 @@ "node": "*" } }, - "node_modules/mlly": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz", - "integrity": "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.15.0", - "pathe": "^2.0.3", - "pkg-types": "^1.3.1", - "ufo": "^1.6.1" - } - }, - "node_modules/mlly/node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2285,9 +2720,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", "dev": true, "funding": [ { @@ -2310,49 +2745,18 @@ "dev": true, "license": "MIT" }, - "node_modules/npm-run-path": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", - "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/npm-run-path/node_modules/path-key": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", - "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/onetime": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", - "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "node_modules/obug": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.3.tgz", + "integrity": "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==", "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], "license": "MIT", - "dependencies": { - "mimic-fn": "^4.0.0" - }, "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=12.20.0" } }, "node_modules/optionator": { @@ -2439,22 +2843,12 @@ } }, "node_modules/pathe": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", - "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", "dev": true, "license": "MIT" }, - "node_modules/pathval": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", - "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -2462,29 +2856,23 @@ "dev": true, "license": "ISC" }, - "node_modules/pkg-types": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", - "integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==", + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", - "dependencies": { - "confbox": "^0.1.8", - "mlly": "^1.7.4", - "pathe": "^2.0.1" + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/pkg-types/node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, "node_modules/postcss": { - "version": "8.5.6", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", - "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "version": "8.5.16", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", + "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", "dev": true, "funding": [ { @@ -2502,7 +2890,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.11", + "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -2520,21 +2908,6 @@ "node": ">= 0.8.0" } }, - "node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -2545,13 +2918,6 @@ "node": ">=6" } }, - "node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -2572,49 +2938,51 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, - "node_modules/rollup": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.57.1.tgz", - "integrity": "sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A==", + "node_modules/rolldown": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.3.tgz", + "integrity": "sha512-1F1eEtUBtFvcGm1HQ9TiUIUHPQG7mSAODrhIzjxoUEFuo8OcbrGLiVLkevNgj84TE4lnHvnumwFjhJO5Eu135g==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "1.0.8" + "@oxc-project/types": "=0.137.0", + "@rolldown/pluginutils": "^1.0.0" }, "bin": { - "rollup": "dist/bin/rollup" + "rolldown": "bin/cli.mjs" }, "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" + "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.57.1", - "@rollup/rollup-android-arm64": "4.57.1", - "@rollup/rollup-darwin-arm64": "4.57.1", - "@rollup/rollup-darwin-x64": "4.57.1", - "@rollup/rollup-freebsd-arm64": "4.57.1", - "@rollup/rollup-freebsd-x64": "4.57.1", - "@rollup/rollup-linux-arm-gnueabihf": "4.57.1", - "@rollup/rollup-linux-arm-musleabihf": "4.57.1", - "@rollup/rollup-linux-arm64-gnu": "4.57.1", - "@rollup/rollup-linux-arm64-musl": "4.57.1", - "@rollup/rollup-linux-loong64-gnu": "4.57.1", - "@rollup/rollup-linux-loong64-musl": "4.57.1", - "@rollup/rollup-linux-ppc64-gnu": "4.57.1", - "@rollup/rollup-linux-ppc64-musl": "4.57.1", - "@rollup/rollup-linux-riscv64-gnu": "4.57.1", - "@rollup/rollup-linux-riscv64-musl": "4.57.1", - "@rollup/rollup-linux-s390x-gnu": "4.57.1", - "@rollup/rollup-linux-x64-gnu": "4.57.1", - "@rollup/rollup-linux-x64-musl": "4.57.1", - "@rollup/rollup-openbsd-x64": "4.57.1", - "@rollup/rollup-openharmony-arm64": "4.57.1", - "@rollup/rollup-win32-arm64-msvc": "4.57.1", - "@rollup/rollup-win32-ia32-msvc": "4.57.1", - "@rollup/rollup-win32-x64-gnu": "4.57.1", - "@rollup/rollup-win32-x64-msvc": "4.57.1", - "fsevents": "~2.3.2" + "@rolldown/binding-android-arm64": "1.1.3", + "@rolldown/binding-darwin-arm64": "1.1.3", + "@rolldown/binding-darwin-x64": "1.1.3", + "@rolldown/binding-freebsd-x64": "1.1.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.3", + "@rolldown/binding-linux-arm64-gnu": "1.1.3", + "@rolldown/binding-linux-arm64-musl": "1.1.3", + "@rolldown/binding-linux-ppc64-gnu": "1.1.3", + "@rolldown/binding-linux-s390x-gnu": "1.1.3", + "@rolldown/binding-linux-x64-gnu": "1.1.3", + "@rolldown/binding-linux-x64-musl": "1.1.3", + "@rolldown/binding-openharmony-arm64": "1.1.3", + "@rolldown/binding-wasm32-wasi": "1.1.3", + "@rolldown/binding-win32-arm64-msvc": "1.1.3", + "@rolldown/binding-win32-x64-msvc": "1.1.3" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" } }, "node_modules/shebang-command": { @@ -2647,19 +3015,6 @@ "dev": true, "license": "ISC" }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -2678,25 +3033,12 @@ "license": "MIT" }, "node_modules/std-env": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", - "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", "dev": true, "license": "MIT" }, - "node_modules/strip-final-newline": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", - "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -2710,19 +3052,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strip-literal": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-2.1.1.tgz", - "integrity": "sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "js-tokens": "^9.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -2743,26 +3072,51 @@ "dev": true, "license": "MIT" }, - "node_modules/tinypool": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.4.tgz", - "integrity": "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ==", + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", "dev": true, "license": "MIT", "engines": { - "node": ">=14.0.0" + "node": ">=18" } }, - "node_modules/tinyspy": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-2.2.1.tgz", - "integrity": "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==", + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", - "engines": { + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { "node": ">=14.0.0" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, "node_modules/tsx": { "version": "4.21.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", @@ -2796,16 +3150,6 @@ "node": ">= 0.8.0" } }, - "node_modules/type-detect": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz", - "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -2820,13 +3164,6 @@ "node": ">=14.17" } }, - "node_modules/ufo": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.3.tgz", - "integrity": "sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==", - "dev": true, - "license": "MIT" - }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -2845,21 +3182,23 @@ } }, "node_modules/vite": { - "version": "5.4.21", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", - "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.0.tgz", + "integrity": "sha512-BuJcQK/56NQTWDGn4ABea3q4SSBdNPWwNZKTkkUpcMPnLoquSYH8llRtSUIgoL1KSCpHt5eghLShn50mH36y7Q==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "^0.21.3", - "postcss": "^8.4.43", - "rollup": "^4.20.0" + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.15", + "rolldown": "~1.1.2", + "tinyglobby": "^0.2.17" }, "bin": { "vite": "bin/vite.js" }, "engines": { - "node": "^18.0.0 || >=20.0.0" + "node": "^20.19.0 || >=22.12.0" }, "funding": { "url": "https://github.com/vitejs/vite?sponsor=1" @@ -2868,23 +3207,33 @@ "fsevents": "~2.3.3" }, "peerDependencies": { - "@types/node": "^18.0.0 || >=20.0.0", - "less": "*", - "lightningcss": "^1.21.0", - "sass": "*", - "sass-embedded": "*", - "stylus": "*", - "sugarss": "*", - "terser": "^5.4.0" + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.3.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" }, "peerDependenciesMeta": { "@types/node": { "optional": true }, - "less": { + "@vitejs/devtools": { "optional": true }, - "lightningcss": { + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { "optional": true }, "sass": { @@ -2901,515 +3250,89 @@ }, "terser": { "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true } } }, - "node_modules/vite-node": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-1.6.1.tgz", - "integrity": "sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA==", - "dev": true, - "license": "MIT", - "dependencies": { - "cac": "^6.7.14", - "debug": "^4.3.4", - "pathe": "^1.1.1", - "picocolors": "^1.0.0", - "vite": "^5.0.0" - }, - "bin": { - "vite-node": "vite-node.mjs" - }, - "engines": { - "node": "^18.0.0 || >=20.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/vite/node_modules/@esbuild/aix-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", - "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", - "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", - "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", - "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", - "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", - "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", - "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", - "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", - "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", - "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", - "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-loong64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", - "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-mips64el": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", - "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", - "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-riscv64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", - "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-s390x": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", - "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", - "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/netbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", - "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/openbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", - "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/sunos-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", - "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", - "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", - "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", - "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/esbuild": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", - "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=12" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.21.5", - "@esbuild/android-arm": "0.21.5", - "@esbuild/android-arm64": "0.21.5", - "@esbuild/android-x64": "0.21.5", - "@esbuild/darwin-arm64": "0.21.5", - "@esbuild/darwin-x64": "0.21.5", - "@esbuild/freebsd-arm64": "0.21.5", - "@esbuild/freebsd-x64": "0.21.5", - "@esbuild/linux-arm": "0.21.5", - "@esbuild/linux-arm64": "0.21.5", - "@esbuild/linux-ia32": "0.21.5", - "@esbuild/linux-loong64": "0.21.5", - "@esbuild/linux-mips64el": "0.21.5", - "@esbuild/linux-ppc64": "0.21.5", - "@esbuild/linux-riscv64": "0.21.5", - "@esbuild/linux-s390x": "0.21.5", - "@esbuild/linux-x64": "0.21.5", - "@esbuild/netbsd-x64": "0.21.5", - "@esbuild/openbsd-x64": "0.21.5", - "@esbuild/sunos-x64": "0.21.5", - "@esbuild/win32-arm64": "0.21.5", - "@esbuild/win32-ia32": "0.21.5", - "@esbuild/win32-x64": "0.21.5" - } - }, "node_modules/vitest": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-1.6.1.tgz", - "integrity": "sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "1.6.1", - "@vitest/runner": "1.6.1", - "@vitest/snapshot": "1.6.1", - "@vitest/spy": "1.6.1", - "@vitest/utils": "1.6.1", - "acorn-walk": "^8.3.2", - "chai": "^4.3.10", - "debug": "^4.3.4", - "execa": "^8.0.1", - "local-pkg": "^0.5.0", - "magic-string": "^0.30.5", - "pathe": "^1.1.1", - "picocolors": "^1.0.0", - "std-env": "^3.5.0", - "strip-literal": "^2.0.0", - "tinybench": "^2.5.1", - "tinypool": "^0.8.3", - "vite": "^5.0.0", - "vite-node": "1.6.1", - "why-is-node-running": "^2.2.2" + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.9.tgz", + "integrity": "sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.9", + "@vitest/mocker": "4.1.9", + "@vitest/pretty-format": "4.1.9", + "@vitest/runner": "4.1.9", + "@vitest/snapshot": "4.1.9", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" }, "bin": { "vitest": "vitest.mjs" }, "engines": { - "node": "^18.0.0 || >=20.0.0" + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "@edge-runtime/vm": "*", - "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "1.6.1", - "@vitest/ui": "1.6.1", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.9", + "@vitest/browser-preview": "4.1.9", + "@vitest/browser-webdriverio": "4.1.9", + "@vitest/coverage-istanbul": "4.1.9", + "@vitest/coverage-v8": "4.1.9", + "@vitest/ui": "4.1.9", "happy-dom": "*", - "jsdom": "*" + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "@edge-runtime/vm": { "optional": true }, + "@opentelemetry/api": { + "optional": true + }, "@types/node": { "optional": true }, - "@vitest/browser": { + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { "optional": true }, "@vitest/ui": { @@ -3420,6 +3343,9 @@ }, "jsdom": { "optional": true + }, + "vite": { + "optional": false } } }, @@ -3476,9 +3402,9 @@ } }, "node_modules/yaml": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", "license": "ISC", "bin": { "yaml": "bin.mjs" diff --git a/tests/package.json b/tests/package.json index 8f9b7b74..82b5fdba 100644 --- a/tests/package.json +++ b/tests/package.json @@ -21,7 +21,7 @@ "zod": "^3.23.8" }, "devDependencies": { - "@microsoft/vally": "^0.6.0", + "@microsoft/vally": "^0.7.0", "@types/node": "^20.14.0", "@vitest/coverage-v8": "^4.1.9", "eslint": "^9.5.0", @@ -32,5 +32,6 @@ }, "engines": { "node": ">=20.0.0" - } -} \ No newline at end of file + }, + "packageManager": "pnpm@11.10.0+sha512.0b7f8b98060031904c017e3a41eb187a16d40eeb829b95c4f8cb03681761fc4ab53dd219115b9b447f4dce1a05a214764461e7d3703392a9f32f9511ce8c86c8" +} diff --git a/tests/pnpm-lock.yaml b/tests/pnpm-lock.yaml index bc23d1e8..cb2b3324 100644 --- a/tests/pnpm-lock.yaml +++ b/tests/pnpm-lock.yaml @@ -25,8 +25,8 @@ importers: version: 3.25.76 devDependencies: '@microsoft/vally': - specifier: ^0.6.0 - version: 0.6.0 + specifier: ^0.7.0 + version: 0.7.0 '@types/node': specifier: ^20.14.0 version: 20.19.30 @@ -47,7 +47,7 @@ importers: version: 8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2) vitest: specifier: ^4.1.9 - version: 4.1.9(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) packages: @@ -291,24 +291,28 @@ packages: resolution: {integrity: sha512-Krg/3ZWxXB7Dw4VOLZEZrYmqc39Yvz9M1K9SPOfjpEy2SFnF/KVLaFt/6E1uYdjgvJ7BmocfVcFYp0hUmy5Axw==} cpu: [arm64] os: [linux] + libc: [glibc] hasBin: true '@github/copilot-linux-x64@1.0.64': resolution: {integrity: sha512-2k9FGppYnxHLwVH+TVCf13JfjSlvS15wsZM5xYxEFlqL/CIBvTK7yn5p7M+P1dWoTUpCmavKG3601BtHPenrRg==} cpu: [x64] os: [linux] + libc: [glibc] hasBin: true '@github/copilot-linuxmusl-arm64@1.0.64': resolution: {integrity: sha512-C+EYoMvmlUxR0YYxLkD3nwn940y0zId8z+pPl9rFO6f9heMGXYCwCZL2i2c3GW6CvGgZF6Wbzw1Kk0Gvw46F2w==} cpu: [arm64] os: [linux] + libc: [musl] hasBin: true '@github/copilot-linuxmusl-x64@1.0.64': resolution: {integrity: sha512-VSTGl7dGQDhH2ACeyYq1hYhAsMUbumpqeVe+CGo/u7fKOMrTNMTR5SB3s+zsHCskKZkJkZ2gsUWROCiDBsThEg==} cpu: [x64] os: [linux] + libc: [musl] hasBin: true '@github/copilot-sdk@1.0.3': @@ -357,8 +361,8 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@microsoft/vally@0.6.0': - resolution: {integrity: sha512-b283YRDFZXUkKNKY3+1EfMBVbHrBLIs5jfUi7lIQ8N0Y10lVsNnNGkRbtTbd7tLYZajr6AhtnpIroc4RFzo1cQ==} + '@microsoft/vally@0.7.0': + resolution: {integrity: sha512-GYxSgub10SC6X1d/vVUCOorGzAMWIUN7uk2zq65pPle0CfKdWy+HiEpnJkgrXGat7kZxzx+jM9lrffW3nfKpmw==} '@napi-rs/wasm-runtime@1.1.5': resolution: {integrity: sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==} @@ -366,6 +370,10 @@ packages: '@emnapi/core': ^1.7.1 '@emnapi/runtime': ^1.7.1 + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + '@oxc-project/types@0.137.0': resolution: {integrity: sha512-WT+Gb24i8hmvo85AIv2oEYouEXkRlKAlT9WaCa3TfLgNCN+GhrJOGZuIlMouAh38Qe4QOx26eUOVsq70qXrywA==} @@ -404,36 +412,42 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.1.2': resolution: {integrity: sha512-LjQP/iZLBu8o8PjIfk4x3At0/mT6h282pvz8Z5LAyhGbu/kDezyO7ea62rF5uoqmgnIYqbN/MqJ3Si3Aymi7xQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [musl] '@rolldown/binding-linux-ppc64-gnu@1.1.2': resolution: {integrity: sha512-X/7bVLWelEsbyWDUSXt7zVsTniLLPIY2n1rH58qr78l9i7MNbbxBWD8gI2vRfBWf4NUXJCUuQnfZDsp32LqsfQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-s390x-gnu@1.1.2': resolution: {integrity: sha512-gb6dYKW/1KDorGXyy48glEBJs/sxVSC5pcVrox/pFGV4mvwSFeg2sK5L2tRkVsVlh7kueqOgg4GEcuipJcGuKg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] + libc: [glibc] '@rolldown/binding-linux-x64-gnu@1.1.2': resolution: {integrity: sha512-JY4w85pU3iAiJVMh5nuk4/Mh9GjMsupe8MrIN53rwxAZW64GKrWeJBuN6SxQg9QTU5uB1cxyhDzW8jqRn1EABw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-x64-musl@1.1.2': resolution: {integrity: sha512-xvpA7o5KCYLB0Rwscmuylb1/zHHSUx4g4xilm4prC5jP76pEUlzBmMbgpbh7bVDbId4NcfT96gN5i6mE6UDaiw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [musl] '@rolldown/binding-openharmony-arm64@1.1.2': resolution: {integrity: sha512-p/ts6KBLjuk49Bp21XH77poQGt02iNz7ChgHep7tudPOaLinR/De/RHdxF8w8Yj4r/bF/bqXwH6PZrB2sA+Nvw==} @@ -822,24 +836,28 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [glibc] lightningcss-linux-arm64-musl@1.32.0: resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [musl] lightningcss-linux-x64-gnu@1.32.0: resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [glibc] lightningcss-linux-x64-musl@1.32.0: resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [musl] lightningcss-win32-arm64-msvc@1.32.0: resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} @@ -1364,9 +1382,10 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@microsoft/vally@0.6.0': + '@microsoft/vally@0.7.0': dependencies: '@github/copilot-sdk': 1.0.3 + '@opentelemetry/api': 1.9.1 js-tiktoken: 1.0.21 picomatch: 4.0.4 yaml: 2.9.0 @@ -1379,6 +1398,8 @@ snapshots: '@tybys/wasm-util': 0.10.3 optional: true + '@opentelemetry/api@1.9.1': {} + '@oxc-project/types@0.137.0': {} '@rolldown/binding-android-arm64@1.1.2': @@ -1466,7 +1487,7 @@ snapshots: obug: 2.1.3 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.9(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) + vitest: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/expect@4.1.9': dependencies: @@ -2001,7 +2022,7 @@ snapshots: tsx: 4.21.0 yaml: 2.8.2 - vitest@4.1.9(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)): + vitest@4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)): dependencies: '@vitest/expect': 4.1.9 '@vitest/mocker': 4.1.9(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) @@ -2024,6 +2045,7 @@ snapshots: vite: 8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: + '@opentelemetry/api': 1.9.1 '@types/node': 20.19.30 '@vitest/coverage-v8': 4.1.9(vitest@4.1.9) transitivePeerDependencies: diff --git a/tests/run-all-evals.ps1 b/tests/run-all-evals.ps1 new file mode 100644 index 00000000..be1e400e --- /dev/null +++ b/tests/run-all-evals.ps1 @@ -0,0 +1,237 @@ +<# +.SYNOPSIS + Runs all Vally evaluations across skill scenarios. + +.DESCRIPTION + Discovers eval.yaml files under the scenarios directory and runs them in parallel + using the Vally evaluation harness. Supports filtering by language and Azure service, + and optional JUnit XML output for CI integration. + +.PARAMETER ScenariosRoot + Path to the directory containing skill scenario folders. + Defaults to: /scenarios + +.PARAMETER ResultsRoot + Path to the directory where evaluation results are written. + Defaults to: /scenario-results + +.PARAMETER Workers + Number of parallel evaluation workers. Default: 5 + +.PARAMETER JUnit + If specified, emit JUnit-compatible XML output alongside the standard results. + +.PARAMETER Language + Filter evaluations to only skills matching the given language suffix(es). + Examples: -Language py, -Language py,ts, -Language rust + +.PARAMETER AzureService + Filter evaluations to only skills matching the given Azure service name(s). + Examples: -AzureService cosmos, -AzureService storage,servicebus + +.EXAMPLE + # Run all evaluations + .\run-all-evals.ps1 + +.EXAMPLE + # Run only Python evaluations + .\run-all-evals.ps1 -Language py + +.EXAMPLE + # Run only Cosmos DB evaluations with JUnit output + .\run-all-evals.ps1 -AzureService cosmos -JUnit + +.EXAMPLE + # Run storage evaluations with 10 parallel workers + .\run-all-evals.ps1 -AzureService storage -Workers 10 + +.EXAMPLE + # Run against a custom scenarios directory + .\run-all-evals.ps1 -ScenariosRoot C:\my-scenarios -ResultsRoot C:\my-results + +.EXAMPLE + # Run with verbose output + .\run-all-evals.ps1 -Verbose + +.EXAMPLE + # Run with debug information + .\run-all-evals.ps1 -Debug + +.NOTES + Supports standard PowerShell switches: -Verbose, -Debug, -InformationAction, -WarningAction, -ErrorAction +#> +[CmdletBinding()] +param( + [Parameter(HelpMessage = "Path to the directory containing skill scenario folders")] + [string]$ScenariosRoot, + + [Parameter(HelpMessage = "Path to the directory where evaluation results are written")] + [string]$ResultsRoot, + + [Parameter(HelpMessage = "Number of parallel evaluation workers")] + [int]$Workers = 5, + + [Parameter(HelpMessage = "Emit JUnit-compatible XML output")] + [switch]$JUnit, + + [Parameter(HelpMessage = "Filter by language suffix(es): py, ts, dotnet, java, rust")] + [string[]]$Language, + + [Parameter(HelpMessage = "Filter by Azure service name(s)")] + [string[]]$AzureService +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +if (-not $PSBoundParameters.ContainsKey("ScenariosRoot")) { + $ScenariosRoot = Join-Path $PSScriptRoot "scenarios" +} + +if (-not $PSBoundParameters.ContainsKey("ResultsRoot")) { + $ResultsRoot = Join-Path $PSScriptRoot "scenario-results" +} + +if (-not (Test-Path -LiteralPath $ScenariosRoot)) { + Write-Error "Scenarios root not found: $ScenariosRoot" + exit 1 +} + +$resolvedScenariosRoot = (Resolve-Path -LiteralPath $ScenariosRoot).ProviderPath +$resolvedResultsRoot = [System.IO.Path]::GetFullPath($ResultsRoot) + +$languageFilter = @($Language | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object { $_.Trim().ToLowerInvariant() }) +$serviceFilter = @($AzureService | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object { $_.Trim().ToLowerInvariant() }) + +$evalFiles = Get-ChildItem -Path $resolvedScenariosRoot -Filter "eval.yaml" -File -Recurse | +Where-Object { + $relativeDir = [System.IO.Path]::GetRelativePath($resolvedScenariosRoot, $_.DirectoryName).TrimStart('\', '/') + if ([string]::IsNullOrWhiteSpace($relativeDir)) { return $false } + + $skillName = ($relativeDir -split '[\\/]', 2)[0] + if ([string]::IsNullOrWhiteSpace($skillName)) { return $false } + + $languageFromSkill = "" + if ($skillName -match '-([^-]+)$') { + $languageFromSkill = $Matches[1].ToLowerInvariant() + } + + $serviceFromSkill = $skillName.ToLowerInvariant() + if ($languageFromSkill) { + $serviceFromSkill = $serviceFromSkill -replace ("-{0}$" -f [regex]::Escape($languageFromSkill)), "" + } + + $includeLanguage = $true + if ($languageFilter.Count -gt 0) { + $includeLanguage = $languageFromSkill -and ($languageFilter -contains $languageFromSkill) + } + + $includeService = $true + if ($serviceFilter.Count -gt 0) { + $includeService = $false + foreach ($svc in $serviceFilter) { + if ($serviceFromSkill -eq $svc -or $serviceFromSkill -like "*$svc*") { + $includeService = $true + break + } + } + } + + return $includeLanguage -and $includeService +} | +Sort-Object FullName + +if (-not $evalFiles) { + $filterSummary = @() + if ($languageFilter.Count -gt 0) { $filterSummary += "language=$($languageFilter -join ',')" } + if ($serviceFilter.Count -gt 0) { $filterSummary += "service=$($serviceFilter -join ',')" } + if ($filterSummary.Count -eq 0) { $filterSummary += "no filters" } + Write-Error "No eval.yaml files found under $resolvedScenariosRoot with $($filterSummary -join '; ')" + exit 1 +} + +New-Item -ItemType Directory -Path $resolvedResultsRoot -Force | Out-Null + +$results = @() + +foreach ($evalFile in $evalFiles) { + $relativeDir = [System.IO.Path]::GetRelativePath($resolvedScenariosRoot, $evalFile.DirectoryName).TrimStart('\', '/') + if ([string]::IsNullOrWhiteSpace($relativeDir)) { $relativeDir = "root" } + + $safeOutDirName = $relativeDir -replace '[\\/:*?"<>|]', "_" + $scenarioOutDir = Join-Path $resolvedResultsRoot $safeOutDirName + New-Item -ItemType Directory -Path $scenarioOutDir -Force | Out-Null + + Write-Host "`n=== Running eval: $($evalFile.FullName) ===" + + $start = Get-Date + $npxArgs = @("vally", "eval", "--eval-spec", $evalFile.FullName, "--output-dir", $scenarioOutDir, "--workers", "$Workers") + if ($JUnit) { $npxArgs += "--junit" } + + & npx @npxArgs + $exitCode = $LASTEXITCODE + $durationSec = [math]::Round(((Get-Date) - $start).TotalSeconds, 1) + + $runDir = Get-ChildItem -Path $scenarioOutDir -Directory -ErrorAction SilentlyContinue | + Sort-Object LastWriteTimeUtc -Descending | + Select-Object -First 1 + + $summary = $null + if ($runDir) { + $jsonlPath = Join-Path $runDir.FullName "results.jsonl" + if (Test-Path -LiteralPath $jsonlPath) { + $line = Get-Content -Path $jsonlPath | + Where-Object { $_ -match '"type"\s*:\s*"run-summary"' } | + Select-Object -Last 1 + if ($line) { + try { $summary = $line | ConvertFrom-Json -Depth 20 } catch {} + } + } + } + + $passed = if ($summary) { [bool]$summary.passed } else { $exitCode -eq 0 } + + $details = "" + if ($summary -and $summary.evals) { + $detailItems = foreach ($e in $summary.evals) { + $status = if ($e.passed) { "PASS" } else { "FAIL" } + if ($e.scoringApplied -and $null -ne $e.overallScore -and $null -ne $e.threshold) { + "{0}:{1} ({2}%/{3}%)" -f $e.name, $status, ([math]::Round([double]$e.overallScore * 100, 1)), ([math]::Round([double]$e.threshold * 100, 1)) + } + elseif ($e.error) { + "{0}:{1} ({2})" -f $e.name, $status, $e.error + } + else { + "{0}:{1}" -f $e.name, $status + } + } + $details = ($detailItems -join "; ") + } + + $results += [pscustomobject]@{ + Scenario = $relativeDir + EvalSpec = $evalFile.FullName + Status = if ($passed) { "PASS" } else { "FAIL" } + ExitCode = $exitCode + DurationSec = $durationSec + RunDir = if ($runDir) { $runDir.FullName } else { "" } + Details = $details + } +} + +Write-Host "`n=== Vally evaluation summary ===" +$results | Sort-Object Scenario | Format-Table -AutoSize Scenario, Status, ExitCode, DurationSec, RunDir + +$reportPath = Join-Path $resolvedResultsRoot ("summary-{0}.csv" -f (Get-Date -Format "yyyyMMdd-HHmmss")) +$results | Export-Csv -NoTypeInformation -Path $reportPath +Write-Host "`nDetailed report saved to: $reportPath" + +$failed = $results | Where-Object { $_.Status -eq "FAIL" } +if ($failed) { + Write-Host "`n=== Failed eval details ===" + $failed | Format-Table -AutoSize Scenario, Details + exit 1 +} + +Write-Host "`nAll evaluations passed." +exit 0 diff --git a/tests/run-harness-by-language.ps1 b/tests/run-harness-by-language.ps1 index 07b33447..d26a6538 100644 --- a/tests/run-harness-by-language.ps1 +++ b/tests/run-harness-by-language.ps1 @@ -18,7 +18,7 @@ If specified, only run mock harness evaluations (skips real Copilot). If specified, only run real Copilot evaluations (skips mock). .PARAMETER ShowDetails -Show detailed output for each scenario. +Show per-scenario details, including generated code and findings. .PARAMETER OutputFile Optional path to save results as JSON. Defaults to stdout only. @@ -113,9 +113,185 @@ $results = @{ } } +function Get-DiagnosticErrorKind { + param( + [string[]]$Diagnostics + ) + + $joined = ($Diagnostics -join "`n").ToLowerInvariant() + + if ($joined -match "authentication failed|bad credentials|github cli user login|401") { + return "auth" + } + if ($joined -match "timeout|session\.idle|timed out") { + return "timeout" + } + if ($joined -match "rate limit|429|econnreset|etimedout|socket hang up|temporarily unavailable|service unavailable") { + return "transient" + } + if ($joined -match "empty or invalid|failed to parse|did not produce json output file") { + return "parse" + } + + return "fatal" +} + +function Test-HarnessAuthPreflight { + if ($env:GH_TOKEN -or $env:GITHUB_TOKEN) { + return $true + } + + try { + $null = & gh auth status 2>&1 + return $LASTEXITCODE -eq 0 + } + catch { + return $false + } +} + +function Get-HarnessModeResult { + param( + [Parameter(Mandatory = $true)] + [string]$Skill, + + [Parameter(Mandatory = $true)] + [bool]$UseMock, + + [switch]$IncludeDetails + ) + + $tempJson = [System.IO.Path]::GetTempFileName() + $args = @("harness", $Skill, "--output", "json", "--output-file", $tempJson) + + if ($UseMock) { + $args += "--mock" + } + + # Capture stdout/stderr as plain strings in case parsing fails and we need diagnostics. + $rawOutput = & pnpm @args 2>&1 | ForEach-Object { "$_" } + $exitCode = $LASTEXITCODE + + try { + if (-not (Test-Path $tempJson)) { + $kind = Get-DiagnosticErrorKind -Diagnostics @($rawOutput) + return @{ + status = "ERROR" + error_kind = $kind + exit_code = $exitCode + error = "Harness did not produce JSON output file." + diagnostics = @($rawOutput) + } + } + + $jsonText = Get-Content $tempJson -Raw + if ([string]::IsNullOrWhiteSpace($jsonText)) { + $kind = Get-DiagnosticErrorKind -Diagnostics @($rawOutput) + return @{ + status = "ERROR" + error_kind = $kind + exit_code = $exitCode + error = "Harness JSON output file was empty." + diagnostics = @($rawOutput) + } + } + + $parsed = $jsonText | ConvertFrom-Json -Depth 100 + + if ($null -eq $parsed) { + $kind = Get-DiagnosticErrorKind -Diagnostics @($rawOutput) + return @{ + status = "ERROR" + error_kind = $kind + exit_code = $exitCode + error = "Harness JSON output was empty or invalid." + diagnostics = @($rawOutput) + } + } + + if ($parsed.status -eq "ERROR") { + $combinedDiagnostics = @($rawOutput) + if ($parsed.diagnostics) { + $combinedDiagnostics += @($parsed.diagnostics) + } + + return @{ + status = "ERROR" + error_kind = if ($parsed.error_kind) { $parsed.error_kind } else { Get-DiagnosticErrorKind -Diagnostics $combinedDiagnostics } + exit_code = $exitCode + error = if ($parsed.error) { $parsed.error } else { "Harness reported an execution error." } + diagnostics = $combinedDiagnostics + } + } + + $result = @{ + status = if ($exitCode -eq 0) { "PASS" } else { "FAIL" } + exit_code = $exitCode + summary = @{ + total_scenarios = $parsed.total_scenarios + passed = $parsed.passed + failed = $parsed.failed + pass_rate = $parsed.pass_rate + avg_score = $parsed.avg_score + duration_ms = $parsed.duration_ms + } + failed_scenarios = @() + } + + foreach ($scenarioResult in @($parsed.results)) { + if (-not $scenarioResult.passed) { + $result.failed_scenarios += @{ + scenario = $scenarioResult.scenario + score = $scenarioResult.score + findings = @($scenarioResult.findings) + } + } + } + + if ($IncludeDetails) { + $result.scenarios = @() + foreach ($scenarioResult in @($parsed.results)) { + $result.scenarios += @{ + scenario = $scenarioResult.scenario + passed = $scenarioResult.passed + score = $scenarioResult.score + generated_code = $scenarioResult.generated_code + raw_response = $scenarioResult.raw_response + findings = @($scenarioResult.findings) + } + } + } + + return $result + } + catch { + $kind = Get-DiagnosticErrorKind -Diagnostics @($rawOutput) + return @{ + status = "ERROR" + error_kind = $kind + exit_code = $exitCode + error = "Failed to parse harness JSON output: $($_.Exception.Message)" + diagnostics = @($rawOutput) + } + } + finally { + Remove-Item -Path $tempJson -Force -ErrorAction SilentlyContinue + } +} + # Run harness for each skill Set-Location $testsDir +$canRunRealHarness = $true +if ($runReal) { + $canRunRealHarness = Test-HarnessAuthPreflight + if (-not $canRunRealHarness) { + Write-Host "Real harness auth preflight failed. Skipping all real-mode runs." -ForegroundColor Red + Write-Host "Run 'gh auth login' or set GH_TOKEN/GITHUB_TOKEN, then rerun." -ForegroundColor Yellow + Write-Host "" + } +} + foreach ($skill in $skills) { $skillResult = @{ name = $skill @@ -130,29 +306,25 @@ foreach ($skill in $skills) { if ($runMock) { Write-Host " [MOCK] Running..." -ForegroundColor Gray -NoNewline try { - $mockOutput = & pnpm harness $skill --mock 2>&1 - $mockSuccess = $LASTEXITCODE -eq 0 + $mockResult = Get-HarnessModeResult -Skill $skill -UseMock $true -IncludeDetails:$ShowDetails + $mockSuccess = $mockResult.status -eq "PASS" if ($mockSuccess) { Write-Host " ✓ PASS" -ForegroundColor Green $results.summary.mock.passed++ - $skillResult.mock = @{ - status = "PASS" - output = $mockOutput - } + $skillResult.mock = $mockResult } else { - Write-Host " ✗ FAIL" -ForegroundColor Red + $statusColor = if ($mockResult.status -eq "ERROR") { "Red" } else { "Red" } + Write-Host " ✗ $($mockResult.status)" -ForegroundColor $statusColor $results.summary.mock.failed++ $results.summary.mock.errors += $skill - $skillResult.mock = @{ - status = "FAIL" - output = $mockOutput - } + $skillResult.mock = $mockResult } - if ($ShowDetails) { - Write-Host $mockOutput -ForegroundColor DarkGray + if ($ShowDetails -and $skillResult.mock.summary) { + $mockSummary = $skillResult.mock.summary + Write-Host " scenarios: $($mockSummary.passed)/$($mockSummary.total_scenarios), avg score: $([math]::Round([double]$mockSummary.avg_score, 1))" -ForegroundColor DarkGray } } catch { @@ -168,31 +340,41 @@ foreach ($skill in $skills) { # Real harness if ($runReal) { + if (-not $canRunRealHarness) { + Write-Host " [REAL] Skipped (auth preflight failed)" -ForegroundColor Red + $results.summary.real.failed++ + $results.summary.real.errors += "$skill (AUTH_ERROR)" + $skillResult.real = @{ + status = "ERROR" + error_kind = "auth" + error = "Skipped due to failed auth preflight. Run 'gh auth login' or set GH_TOKEN/GITHUB_TOKEN." + } + Write-Host "" + $results.skills += $skillResult + continue + } + Write-Host " [REAL] Running..." -ForegroundColor Gray -NoNewline try { - $realOutput = & pnpm harness $skill 2>&1 - $realSuccess = $LASTEXITCODE -eq 0 + $realResult = Get-HarnessModeResult -Skill $skill -UseMock $false -IncludeDetails:$ShowDetails + $realSuccess = $realResult.status -eq "PASS" if ($realSuccess) { Write-Host " ✓ PASS" -ForegroundColor Green $results.summary.real.passed++ - $skillResult.real = @{ - status = "PASS" - output = $realOutput - } + $skillResult.real = $realResult } else { - Write-Host " ✗ FAIL" -ForegroundColor Red + $statusColor = if ($realResult.status -eq "ERROR") { "Red" } else { "Red" } + Write-Host " ✗ $($realResult.status)" -ForegroundColor $statusColor $results.summary.real.failed++ $results.summary.real.errors += $skill - $skillResult.real = @{ - status = "FAIL" - output = $realOutput - } + $skillResult.real = $realResult } - if ($ShowDetails) { - Write-Host $realOutput -ForegroundColor DarkGray + if ($ShowDetails -and $skillResult.real.summary) { + $realSummary = $skillResult.real.summary + Write-Host " scenarios: $($realSummary.passed)/$($realSummary.total_scenarios), avg score: $([math]::Round([double]$realSummary.avg_score, 1))" -ForegroundColor DarkGray } } catch { diff --git a/tests/run-skill-experiments.ps1 b/tests/run-skill-experiments.ps1 new file mode 100644 index 00000000..3820535a --- /dev/null +++ b/tests/run-skill-experiments.ps1 @@ -0,0 +1,1636 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS +Run vally skill effectiveness experiments for specified skills. + +.DESCRIPTION +Executes skill_effectiveness_experiment.yaml files for all or specified skills. +Each experiment runs two variants (baseline without skill, skill with skill context) and +stores results organized by timestamp. + +Use -SkillPattern to filter by language (e.g., '*-py' for Python, '*-rs' for Rust) +or by service (e.g., 'azure-cosmos*' for Cosmos DB). + +Results can be compared using compare-experiment.mjs to analyze skill impact on performance. + +.PARAMETER ScenariosRoot +Root directory containing skill scenario subdirectories. +Defaults to: /scenarios + +.PARAMETER ResultsRoot +Root directory where experiment results will be stored. +Defaults to: /vally-experiment-results + +.PARAMETER SkillPattern +Filter which skills to run. Supports wildcards. +Defaults to '*-py' (all Python skills). +Examples: -SkillPattern '*-rust', -SkillPattern 'azure-ai-*' + +.PARAMETER Workers +Number of parallel experiment runs. Defaults to 1 (sequential). + +.PARAMETER DryRun +If set, shows what experiments would be run without actually running them. + +.PARAMETER Compare +If set, runs compare-experiment.mjs after completion to generate A/B reports. + +.PARAMETER AnalysisOnly +If set, skips vally execution and regenerates narratives/comparisons from existing outputs. + +.PARAMETER ExperimentOutputDir +Existing experiment output root to analyze when using -AnalysisOnly. +If omitted in analysis-only mode, the latest directory under ResultsRoot is used. + +.PARAMETER CopilotMaxAttempts +Maximum retries per copilot invocation path for narrative generation. + +.PARAMETER NoNodeLoaderFallback +Disable node-loader fallback and use only documented copilot invocation methods. + +.EXAMPLE +./run-skill-experiments.ps1 +# Run all Python skill experiments sequentially + +.EXAMPLE +./run-skill-experiments.ps1 -SkillPattern '*-rust' -Workers 4 +# Run all Rust skill experiments with 4 parallel workers + +.EXAMPLE +./run-skill-experiments.ps1 -SkillPattern 'azure-cosmos*' -Verbose +# Run only Cosmos DB skill experiments with verbose output + +.EXAMPLE +./run-skill-experiments.ps1 -Workers 5 -Compare +# Run all (Python) experiments with 5 parallel workers, then generate comparison reports + +.EXAMPLE +./run-skill-experiments.ps1 -DryRun +# Preview which experiments would be run + +.EXAMPLE +./run-skill-experiments.ps1 -AnalysisOnly -ExperimentOutputDir "q:\src\skills\tests\vally-experiment-results\2026-07-06T17-00-00-000Z" +# Regenerate narratives/comparisons from an existing experiment output without rerunning vally +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ScenariosRoot = (Join-Path $PSScriptRoot "scenarios"), + + [Parameter(Mandatory = $false)] + [string]$ResultsRoot = (Join-Path $PSScriptRoot "vally-experiment-results"), + + [Parameter(Mandatory = $false)] + [string]$SkillPattern = "*-py", + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 128)] + [int]$Workers = 1, + + [Parameter(Mandatory = $false)] + [switch]$DryRun, + + [Parameter(Mandatory = $false)] + [switch]$Compare + + , + [Parameter(Mandatory = $false)] + [switch]$AnalysisOnly + + , + [Parameter(Mandatory = $false)] + [string]$ExperimentOutputDir + + , + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10)] + [int]$CopilotMaxAttempts = 2, + + [Parameter(Mandatory = $false)] + [switch]$NoNodeLoaderFallback +) + +$ErrorActionPreference = "Stop" + +function Get-ExperimentOutputDirectory { + param( + [Parameter(Mandatory = $true)] + [string]$OutputText + ) + + $outputMatches = [regex]::Matches($OutputText, 'Output:\s*(.+)') + if ($outputMatches.Count -eq 0) { + return $null + } + + for ($i = $outputMatches.Count - 1; $i -ge 0; $i--) { + $candidate = $outputMatches[$i].Groups[1].Value.Trim() + if (Test-Path $candidate) { + return $candidate + } + } + + return $outputMatches[$outputMatches.Count - 1].Groups[1].Value.Trim() +} + +function Get-VariantDigest { + param( + [Parameter(Mandatory = $true)] + [string]$ResultsJsonlPath + ) + + if (-not (Test-Path $ResultsJsonlPath)) { + return $null + } + + $records = @() + foreach ($line in (Get-Content -Path $ResultsJsonlPath -ErrorAction SilentlyContinue)) { + if ([string]::IsNullOrWhiteSpace($line)) { + continue + } + + try { + $obj = $line | ConvertFrom-Json + if ($obj.type -eq "trial-result") { + $records += $obj + } + } + catch { + continue + } + } + + if ($records.Count -eq 0) { + return $null + } + + $variantName = $records[0].variant + $evalName = $records[0].evalName + $model = $records[0].model + + $scores = @() + $tokens = @() + $turns = @() + $toolCalls = @() + $wallTimesMs = @() + $errors = @() + $skillActivations = @() + $toolBreakdownTotals = @{} + $skillBreakdownTotals = @{} + $failedGraderBreakdown = @{} + $timeoutGraderBreakdown = @{} + $totalFailedGraders = 0 + $totalTimeoutGraders = 0 + $stimuli = @() + + foreach ($record in $records) { + if ($record.gradeResult -and $null -ne $record.gradeResult.score) { + $scores += [double]$record.gradeResult.score + } + + $traj = $record.trajectory + $metrics = $traj.metrics + if ($metrics) { + if ($metrics.tokenUsage -and $null -ne $metrics.tokenUsage.totalTokens) { + $tokens += [int]$metrics.tokenUsage.totalTokens + } + if ($null -ne $metrics.turnCount) { + $turns += [int]$metrics.turnCount + } + if ($null -ne $metrics.toolCallCount) { + $toolCalls += [int]$metrics.toolCallCount + } + if ($null -ne $metrics.wallTimeMs) { + $wallTimesMs += [int]$metrics.wallTimeMs + } + if ($null -ne $metrics.errorCount) { + $errors += [int]$metrics.errorCount + } + if ($null -ne $metrics.skillActivationCount) { + $skillActivations += [int]$metrics.skillActivationCount + } + + if ($metrics.toolCallBreakdown) { + $metrics.toolCallBreakdown.psobject.Properties | ForEach-Object { + if (-not $toolBreakdownTotals.ContainsKey($_.Name)) { + $toolBreakdownTotals[$_.Name] = 0 + } + $toolBreakdownTotals[$_.Name] += [int]$_.Value + } + } + + if ($metrics.skillActivationBreakdown) { + $metrics.skillActivationBreakdown.psobject.Properties | ForEach-Object { + if (-not $skillBreakdownTotals.ContainsKey($_.Name)) { + $skillBreakdownTotals[$_.Name] = 0 + } + $skillBreakdownTotals[$_.Name] += [int]$_.Value + } + } + } + + $stimulus = $traj.stimulus + $graderOutcomes = @() + if ($record.gradeResult -and $record.gradeResult.details) { + foreach ($detail in $record.gradeResult.details) { + $graderEvidence = if ($null -ne $detail.evidence) { [string]$detail.evidence } else { "" } + $isTimeout = (-not [bool]$detail.passed) -and ($graderEvidence -match '(?i)\btime(d)?\s*out\b|\btimeout\b') + + if (-not [bool]$detail.passed) { + $totalFailedGraders++ + + if (-not $failedGraderBreakdown.ContainsKey($detail.name)) { + $failedGraderBreakdown[$detail.name] = 0 + } + $failedGraderBreakdown[$detail.name] += 1 + + if ($isTimeout) { + $totalTimeoutGraders++ + if (-not $timeoutGraderBreakdown.ContainsKey($detail.name)) { + $timeoutGraderBreakdown[$detail.name] = 0 + } + $timeoutGraderBreakdown[$detail.name] += 1 + } + } + + $graderOutcomes += [ordered]@{ + name = $detail.name + passed = [bool]$detail.passed + score = if ($null -ne $detail.score) { [double]$detail.score } else { $null } + evidence = $detail.evidence + isTimeout = $isTimeout + } + } + } + + $outputPreview = "" + if ($traj -and $traj.output) { + $outputString = [string]$traj.output + $maxLen = 800 + $outputPreview = if ($outputString.Length -gt $maxLen) { $outputString.Substring(0, $maxLen) } else { $outputString } + } + + $stimuli += [ordered]@{ + name = $record.stimulus + passed = [bool]$record.gradeResult.passed + scorePct = if ($null -ne $record.gradeResult.score) { [math]::Round(([double]$record.gradeResult.score * 100), 1) } else { $null } + durationSec = [math]::Round(([double]$record.durationMs / 1000), 1) + prompt = if ($stimulus) { $stimulus.prompt } else { $null } + rubric = if ($stimulus) { $stimulus.rubric } else { @() } + toolCallBreakdown = if ($metrics) { $metrics.toolCallBreakdown } else { $null } + graderOutcomes = $graderOutcomes + outputPreview = $outputPreview + } + } + + return [ordered]@{ + variant = $variantName + eval = $evalName + model = $model + trialCount = $records.Count + avgScorePct = if ($scores.Count -gt 0) { [math]::Round((($scores | Measure-Object -Average).Average * 100), 1) } else { $null } + avgTokens = if ($tokens.Count -gt 0) { [int](($tokens | Measure-Object -Average).Average) } else { $null } + avgTurns = if ($turns.Count -gt 0) { [math]::Round((($turns | Measure-Object -Average).Average), 1) } else { $null } + avgToolCalls = if ($toolCalls.Count -gt 0) { [math]::Round((($toolCalls | Measure-Object -Average).Average), 1) } else { $null } + avgWallTimeSec = if ($wallTimesMs.Count -gt 0) { [math]::Round(((($wallTimesMs | Measure-Object -Average).Average) / 1000), 1) } else { $null } + totalErrors = if ($errors.Count -gt 0) { [int](($errors | Measure-Object -Sum).Sum) } else { 0 } + totalSkillActivations = if ($skillActivations.Count -gt 0) { [int](($skillActivations | Measure-Object -Sum).Sum) } else { 0 } + toolCallBreakdownTotals = $toolBreakdownTotals + skillActivationBreakdownTotals = $skillBreakdownTotals + totalFailedGraders = $totalFailedGraders + totalTimeoutGraders = $totalTimeoutGraders + failedGraderBreakdown = $failedGraderBreakdown + timeoutGraderBreakdown = $timeoutGraderBreakdown + stimuli = $stimuli + } +} + +function Invoke-ExternalProcessCaptured { + param( + [Parameter(Mandatory = $true)] + [string]$FilePath, + + [Parameter(Mandatory = $false)] + [string[]]$ArgumentList = @() + ) + + try { + $resolvedPath = $FilePath + $resolvedCommand = Get-Command -Name $FilePath -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($resolvedCommand -and -not [string]::IsNullOrWhiteSpace($resolvedCommand.Source)) { + $resolvedPath = $resolvedCommand.Source + } + + $effectiveFilePath = $resolvedPath + $effectiveArgs = New-Object System.Collections.Generic.List[string] + foreach ($arg in $ArgumentList) { + [void]$effectiveArgs.Add([string]$arg) + } + + if ($resolvedPath -and $resolvedPath.EndsWith('.ps1', [System.StringComparison]::OrdinalIgnoreCase)) { + $pwshCommand = Get-Command -Name pwsh -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $pwshCommand) { + $pwshCommand = Get-Command -Name powershell -ErrorAction SilentlyContinue | Select-Object -First 1 + } + + if (-not $pwshCommand -or [string]::IsNullOrWhiteSpace($pwshCommand.Source)) { + throw "PowerShell executable not found while attempting to launch script: $resolvedPath" + } + + $effectiveFilePath = $pwshCommand.Source + $scriptArgs = New-Object System.Collections.Generic.List[string] + [void]$scriptArgs.Add('-NoProfile') + [void]$scriptArgs.Add('-File') + [void]$scriptArgs.Add($resolvedPath) + foreach ($arg in $effectiveArgs) { + [void]$scriptArgs.Add($arg) + } + $effectiveArgs = $scriptArgs + } + + $psi = [System.Diagnostics.ProcessStartInfo]::new() + $psi.FileName = $effectiveFilePath + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + + foreach ($arg in $effectiveArgs) { + [void]$psi.ArgumentList.Add([string]$arg) + } + + $process = [System.Diagnostics.Process]::new() + $process.StartInfo = $psi + + [void]$process.Start() + $stdoutText = $process.StandardOutput.ReadToEnd() + $stderrText = $process.StandardError.ReadToEnd() + $process.WaitForExit() + + $combinedOutput = $stdoutText + if (-not [string]::IsNullOrWhiteSpace($stderrText)) { + if (-not [string]::IsNullOrWhiteSpace($combinedOutput)) { + $combinedOutput += "`n" + } + $combinedOutput += $stderrText + } + + return [pscustomobject]@{ + ExitCode = $process.ExitCode + Output = $combinedOutput.Trim() + Stderr = $stderrText.Trim() + } + } + catch { + return [pscustomobject]@{ + ExitCode = -1 + Output = "" + Stderr = $_.Exception.Message + } + } +} + +function Invoke-CopilotNarrative { + param( + [Parameter(Mandatory = $true)] + [string]$Prompt, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10)] + [int]$MaxAttempts = 2, + + [Parameter(Mandatory = $false)] + [bool]$EnableNodeLoaderFallback = $true + ) + + $copilotCommand = Get-Command copilot -ErrorAction SilentlyContinue + if (-not $copilotCommand) { + return $null + } + + $copilotSource = $copilotCommand.Source + $copilotDir = Split-Path -Path $copilotSource -Parent + + $npmLoaderPath = $null + $loaderCandidates = @( + (Join-Path $copilotDir "node_modules\@github\copilot\npm-loader.js"), + "q:\.tools\.npm-global\node_modules\@github\copilot\npm-loader.js" + ) + + try { + $npmGlobalRoot = (& npm root -g 2>$null | Select-Object -First 1) + if (-not [string]::IsNullOrWhiteSpace($npmGlobalRoot)) { + $loaderCandidates += (Join-Path $npmGlobalRoot.Trim() "@github\copilot\npm-loader.js") + } + } + catch { + # Ignore npm-root discovery failures and continue with known candidates. + } + + foreach ($candidate in $loaderCandidates) { + if (-not [string]::IsNullOrWhiteSpace($candidate) -and (Test-Path $candidate)) { + $npmLoaderPath = $candidate + break + } + } + + $baseArgs = @("--silent", "--allow-all-tools", "--allow-all-paths", "-p", $Prompt) + + $attempts = @() + + $npxInvoker = $null + $npxCandidates = @("npx.ps1", "npx.cmd", "npx") + foreach ($candidate in $npxCandidates) { + $npxCommand = Get-Command -Name $candidate -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($npxCommand -and -not [string]::IsNullOrWhiteSpace($npxCommand.Source)) { + $npxInvoker = $npxCommand.Source + break + } + } + + if (-not $npxInvoker) { + $npxInvoker = "npx" + } + + # First choice: documented npx invocation. + $attempts += [pscustomobject]@{ + Name = "npx" + Cmd = $npxInvoker + Args = @("@github/copilot") + $baseArgs + } + + # Final fallback: use discovered copilot command path (prefer .cmd over .ps1). + $fallbackInvoker = $copilotSource + if ($fallbackInvoker -and $fallbackInvoker.EndsWith('.ps1', [System.StringComparison]::OrdinalIgnoreCase)) { + $cmdCandidate = [System.IO.Path]::ChangeExtension($fallbackInvoker, '.cmd') + if (Test-Path $cmdCandidate) { + $fallbackInvoker = $cmdCandidate + } + } + + $attempts += [pscustomobject]@{ + Name = "copilot-wrapper" + Cmd = $fallbackInvoker + Args = $baseArgs + } + + # Final fallback: npm-loader.js via node (undocumented implementation detail). + if ($EnableNodeLoaderFallback -and (Test-Path $npmLoaderPath)) { + $attempts += [pscustomobject]@{ + Name = "node-loader" + Cmd = "node" + Args = @($npmLoaderPath) + $baseArgs + } + } + + foreach ($attempt in $attempts) { + for ($i = 1; $i -le $MaxAttempts; $i++) { + Write-Verbose "Copilot narrative attempt '$($attempt.Name)' try $i/$MaxAttempts" + $execution = Invoke-ExternalProcessCaptured -FilePath $attempt.Cmd -ArgumentList @($attempt.Args) + + if ($execution.ExitCode -eq 0) { + Write-Verbose "Copilot narrative succeeded via '$($attempt.Name)'" + return $execution.Output + } + + if (-not [string]::IsNullOrWhiteSpace($execution.Stderr)) { + Write-Verbose "Copilot narrative '$($attempt.Name)' failed: $($execution.Stderr)" + } + else { + Write-Verbose "Copilot narrative '$($attempt.Name)' exited with code $($execution.ExitCode)" + } + + if ($i -lt $MaxAttempts) { + Start-Sleep -Milliseconds (250 * $i) + } + } + } + + return $null +} + +function Resolve-ExperimentOutputDir { + param( + [Parameter(Mandatory = $true)] + [string]$SkillDirPath + ) + + if (-not (Test-Path $SkillDirPath)) { + return $null + } + + $directVariantDirs = Get-ChildItem -Path $SkillDirPath -Directory -ErrorAction SilentlyContinue | Where-Object { + Test-Path (Join-Path $_.FullName "results.jsonl") + } + + if ($directVariantDirs.Count -ge 2) { + return $SkillDirPath + } + + $candidateRuns = Get-ChildItem -Path $SkillDirPath -Directory -ErrorAction SilentlyContinue | Sort-Object Name -Descending + foreach ($candidate in $candidateRuns) { + $variantDirs = Get-ChildItem -Path $candidate.FullName -Directory -ErrorAction SilentlyContinue | Where-Object { + Test-Path (Join-Path $_.FullName "results.jsonl") + } + if ($variantDirs.Count -ge 2) { + return $candidate.FullName + } + } + + return $null +} + +function Get-LatestResultsDirectory { + param( + [Parameter(Mandatory = $true)] + [string]$RootPath + ) + + if (-not (Test-Path $RootPath)) { + return $null + } + + $dirs = Get-ChildItem -Path $RootPath -Directory -ErrorAction SilentlyContinue | Sort-Object Name -Descending + if ($dirs.Count -eq 0) { + return $null + } + + return $dirs[0].FullName +} + +function Generate-NarrativesOnly { + param( + [Parameter(Mandatory = $true)] + [string]$AnalysisRoot, + + [Parameter(Mandatory = $true)] + [string]$SkillNamePattern, + + [Parameter(Mandatory = $false)] + [int]$NarrativeMaxAttempts = 2, + + [Parameter(Mandatory = $false)] + [bool]$EnableNodeLoaderFallback = $true + ) + + Write-Information "" + Write-Information "Analysis-Only Mode" + Write-Information "Source Results Directory: $AnalysisRoot" + + $skillDirs = Get-ChildItem -Path $AnalysisRoot -Directory -Filter $SkillNamePattern -ErrorAction SilentlyContinue | Sort-Object Name + if ($skillDirs.Count -eq 0) { + Write-Warning "No skill result directories found in $AnalysisRoot matching pattern: $SkillNamePattern" + return 1 + } + + $generated = 0 + $failed = 0 + + foreach ($skillDir in $skillDirs) { + $experimentDir = Resolve-ExperimentOutputDir -SkillDirPath $skillDir.FullName + if (-not $experimentDir) { + Write-Host " ✗ No valid experiment output found for $($skillDir.Name)" -ForegroundColor Yellow + $failed++ + continue + } + + $ok = Generate-SkillNarratives -SkillName $skillDir.Name -ExperimentOutputDir $experimentDir -SkillResultsDir $skillDir.FullName -NarrativeMaxAttempts $NarrativeMaxAttempts -EnableNodeLoaderFallback $EnableNodeLoaderFallback + if ($ok) { + $generated++ + Write-Host " ✓ Narratives generated for $($skillDir.Name)" -ForegroundColor Green + } + else { + $failed++ + Write-Host " ✗ Narrative generation failed for $($skillDir.Name)" -ForegroundColor Yellow + continue + } + } + + Write-Host "Narrative Summary: generated=$generated failed=$failed" -ForegroundColor Cyan + if ($failed -gt 0) { + return 1 + } + return 0 +} + +function Normalize-NarrativeText { + param( + [Parameter(Mandatory = $true)] + [string]$Text + ) + + # Keep markdown output plain ASCII to avoid rendering/encoding mismatches. + $normalized = $Text + $normalized = $normalized -replace '[^\x09\x0A\x0D\x20-\x7E]', '' + + return $normalized +} + +function Convert-ToolBreakdownToText { + param( + [Parameter(Mandatory = $false)] + $Breakdown + ) + + if (-not $Breakdown) { + return "none" + } + + $parts = @() + if ($Breakdown -is [System.Collections.IDictionary]) { + foreach ($key in $Breakdown.Keys) { + $parts += "${key}: $($Breakdown[$key])" + } + } + else { + $Breakdown.psobject.Properties | ForEach-Object { + $parts += "$($_.Name): $($_.Value)" + } + } + + if ($parts.Count -eq 0) { + return "none" + } + + return ($parts -join ", ") +} + +function Get-PromptFocusSummary { + param( + [Parameter(Mandatory = $false)] + [string]$Prompt + ) + + if ([string]::IsNullOrWhiteSpace($Prompt)) { + return "Prompt text not available in digest." + } + + $singleLine = ($Prompt -replace "`r?`n", " ").Trim() + $maxLen = 260 + if ($singleLine.Length -le $maxLen) { + return $singleLine + } + + return $singleLine.Substring(0, $maxLen).Trim() + "..." +} + +function Format-ElapsedDuration { + param( + [Parameter(Mandatory = $true)] + [double]$Seconds + ) + + $duration = [TimeSpan]::FromSeconds($Seconds) + return "{0}h {1}m {2:F1}s" -f [int]$duration.TotalHours, $duration.Minutes, ($duration.Seconds + ($duration.Milliseconds / 1000)) +} + +function New-VariantNarrativeFromDigest { + param( + [Parameter(Mandatory = $true)] + $Digest + ) + + $sb = New-Object System.Text.StringBuilder + $failedGraderNamesText = if ($Digest.failedGraderBreakdown -and $Digest.failedGraderBreakdown.Count -gt 0) { ($Digest.failedGraderBreakdown.Keys | Sort-Object) -join ", " } else { "none" } + $timeoutGraderNamesText = if ($Digest.timeoutGraderBreakdown -and $Digest.timeoutGraderBreakdown.Count -gt 0) { ($Digest.timeoutGraderBreakdown.Keys | Sort-Object) -join ", " } else { "none" } + + [void]$sb.AppendLine("# Execution Summary") + [void]$sb.AppendLine() + [void]$sb.AppendLine("Variant $($Digest.variant) in eval $($Digest.eval) with model $($Digest.model) ran $($Digest.trialCount) trials.") + [void]$sb.AppendLine("Average metrics for this variant: score $($Digest.avgScorePct)%, tokens $($Digest.avgTokens), turns $($Digest.avgTurns), tool calls $($Digest.avgToolCalls), wall time $($Digest.avgWallTimeSec)s.") + [void]$sb.AppendLine("Run totals for this variant: errors $($Digest.totalErrors), skill activations $($Digest.totalSkillActivations), tool usage $(Convert-ToolBreakdownToText -Breakdown $Digest.toolCallBreakdownTotals).") + [void]$sb.AppendLine("Grader outcomes across this variant: failed graders $($Digest.totalFailedGraders), timeout-related grader failures $($Digest.totalTimeoutGraders).") + [void]$sb.AppendLine("Failed grader names: $failedGraderNamesText. Timeout-related grader names: $timeoutGraderNamesText.") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Stimulus Narratives") + [void]$sb.AppendLine() + + foreach ($stimulus in $Digest.stimuli) { + [void]$sb.AppendLine("### Stimulus: $($stimulus.name)") + [void]$sb.AppendLine("- Prompt Focus: $(Get-PromptFocusSummary -Prompt $stimulus.prompt)") + [void]$sb.AppendLine("- Actions Taken: turns $($Digest.avgTurns) average for this variant; tool calls for this stimulus were $(Convert-ToolBreakdownToText -Breakdown $stimulus.toolCallBreakdown).") + + $graderParts = @() + foreach ($grader in $stimulus.graderOutcomes) { + $graderStatus = if ($grader.passed) { "passed" } else { "failed" } + $graderParts += "$($grader.name): $graderStatus ($($grader.score))" + } + $graderText = if ($graderParts.Count -gt 0) { $graderParts -join "; " } else { "none" } + + $resultStatus = if ($stimulus.passed) { "passed" } else { "failed" } + [void]$sb.AppendLine("- Result: stimulus $resultStatus with score $($stimulus.scorePct)% in $($stimulus.durationSec)s; graders: $graderText.") + + $failedGraders = @($stimulus.graderOutcomes | Where-Object { -not $_.passed }) + $timeoutGraders = @($failedGraders | Where-Object { $_.isTimeout }) + $nonTimeoutGraders = @($failedGraders | Where-Object { -not $_.isTimeout }) + + if ($failedGraders.Count -eq 0) { + [void]$sb.AppendLine("- Failed graders: none.") + } + else { + $nonTimeoutText = if ($nonTimeoutGraders.Count -gt 0) { ($nonTimeoutGraders | ForEach-Object { $_.name } | Sort-Object -Unique) -join ", " } else { "none" } + $timeoutText = if ($timeoutGraders.Count -gt 0) { ($timeoutGraders | ForEach-Object { $_.name } | Sort-Object -Unique) -join ", " } else { "none" } + [void]$sb.AppendLine("- Failed graders: non-timeout = $nonTimeoutText; timeout-related = $timeoutText.") + if ($timeoutGraders.Count -gt 0) { + [void]$sb.AppendLine("- Timeout note: one or more grader failures indicate timeout behavior; consider checking whether the stimulus timeout budget is too low for this scenario.") + } + } + [void]$sb.AppendLine() + } + + [void]$sb.AppendLine("# Tool and Turn Behavior") + [void]$sb.AppendLine() + [void]$sb.AppendLine("This variant averaged $($Digest.avgTurns) turns and $($Digest.avgToolCalls) tool calls per trial.") + [void]$sb.AppendLine("Aggregate tool breakdown for this variant: $(Convert-ToolBreakdownToText -Breakdown $Digest.toolCallBreakdownTotals).") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Grading Outcome") + [void]$sb.AppendLine() + [void]$sb.AppendLine("Trials run: $($Digest.trialCount). Average score for this variant: $($Digest.avgScorePct)%.") + [void]$sb.AppendLine("Total errors recorded: $($Digest.totalErrors).") + [void]$sb.AppendLine("Failed grader count: $($Digest.totalFailedGraders). Timeout-related grader failures: $($Digest.totalTimeoutGraders).") + [void]$sb.AppendLine("Failed grader names: $failedGraderNamesText. Timeout-related grader names: $timeoutGraderNamesText.") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Notable Strengths") + [void]$sb.AppendLine() + [void]$sb.AppendLine("- Completed $($Digest.trialCount) trials with total errors $($Digest.totalErrors).") + [void]$sb.AppendLine("- Produced runnable code artifacts validated during each trial.") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Notable Weaknesses") + [void]$sb.AppendLine() + if ([int]$Digest.totalFailedGraders -gt 0) { + [void]$sb.AppendLine("- Failed graders were recorded: $(Convert-ToolBreakdownToText -Breakdown $Digest.failedGraderBreakdown).") + } + else { + [void]$sb.AppendLine("- No failed graders were recorded in this variant.") + } + + if ([int]$Digest.totalTimeoutGraders -gt 0) { + [void]$sb.AppendLine("- Timeout-related grader failures were recorded: $(Convert-ToolBreakdownToText -Breakdown $Digest.timeoutGraderBreakdown). This often points to an undersized stimulus timeout budget rather than a logic error.") + } + + return $sb.ToString().Trim() +} + +function Get-PassRatePct { + param( + [Parameter(Mandatory = $true)] + $Digest + ) + + if (-not $Digest.stimuli -or $Digest.stimuli.Count -eq 0) { + return 0.0 + } + + $passedCount = @($Digest.stimuli | Where-Object { $_.passed }).Count + return [math]::Round((100.0 * $passedCount / $Digest.stimuli.Count), 1) +} + +function Get-OverallEffectLabel { + param( + [Parameter(Mandatory = $true)] + [double]$ScoreDelta, + + [Parameter(Mandatory = $true)] + [double]$PassRateDelta, + + [Parameter(Mandatory = $true)] + [double]$TokenDeltaPct, + + [Parameter(Mandatory = $true)] + [double]$WallTimeDeltaPct + ) + + $qualityImproved = ($ScoreDelta -ge 1.0) -or ($PassRateDelta -gt 0) + $qualityRegressed = ($ScoreDelta -le -1.0) -or ($PassRateDelta -lt 0) + $efficiencyImproved = ($TokenDeltaPct -le -5.0) -or ($WallTimeDeltaPct -le -5.0) + $efficiencyRegressed = ($TokenDeltaPct -ge 5.0) -or ($WallTimeDeltaPct -ge 5.0) + + if ($qualityImproved -and -not $qualityRegressed -and -not $efficiencyRegressed) { + return "Improved" + } + if ($qualityRegressed -and -not $qualityImproved -and -not $efficiencyImproved) { + return "Regressed" + } + if (($qualityImproved -and $efficiencyRegressed) -or ($qualityRegressed -and $efficiencyImproved)) { + return "Mixed" + } + + return "Neutral" +} + +function New-ComparisonNarrativeFromDigests { + param( + [Parameter(Mandatory = $true)] + [string]$SkillName, + + [Parameter(Mandatory = $true)] + [string]$BaselineVariant, + + [Parameter(Mandatory = $true)] + [string]$SkillVariant, + + [Parameter(Mandatory = $true)] + $BaselineDigest, + + [Parameter(Mandatory = $true)] + $SkillDigest + ) + + $baselineScore = [double]$BaselineDigest.avgScorePct + $skillScore = [double]$SkillDigest.avgScorePct + $scoreDelta = [math]::Round(($skillScore - $baselineScore), 1) + + $baselinePassRate = Get-PassRatePct -Digest $BaselineDigest + $skillPassRate = Get-PassRatePct -Digest $SkillDigest + $passRateDelta = [math]::Round(($skillPassRate - $baselinePassRate), 1) + + $baselineTokens = [double]$BaselineDigest.avgTokens + $skillTokens = [double]$SkillDigest.avgTokens + $tokenDeltaPct = if ($baselineTokens -gt 0) { [math]::Round((($skillTokens - $baselineTokens) / $baselineTokens) * 100.0, 1) } else { 0.0 } + + $baselineWall = [double]$BaselineDigest.avgWallTimeSec + $skillWall = [double]$SkillDigest.avgWallTimeSec + $wallTimeDeltaPct = if ($baselineWall -gt 0) { [math]::Round((($skillWall - $baselineWall) / $baselineWall) * 100.0, 1) } else { 0.0 } + + $overallEffect = Get-OverallEffectLabel -ScoreDelta $scoreDelta -PassRateDelta $passRateDelta -TokenDeltaPct $tokenDeltaPct -WallTimeDeltaPct $wallTimeDeltaPct + + $sb = New-Object System.Text.StringBuilder + + [void]$sb.AppendLine("# Overall Effect") + [void]$sb.AppendLine() + [void]$sb.AppendLine("$overallEffect") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Evidence From Process") + [void]$sb.AppendLine() + [void]$sb.AppendLine("- Turns: $BaselineVariant = $($BaselineDigest.avgTurns), $SkillVariant = $($SkillDigest.avgTurns) (delta $([math]::Round(([double]$SkillDigest.avgTurns - [double]$BaselineDigest.avgTurns), 1))).") + [void]$sb.AppendLine("- Tool calls: $BaselineVariant = $($BaselineDigest.avgToolCalls), $SkillVariant = $($SkillDigest.avgToolCalls) (delta $([math]::Round(([double]$SkillDigest.avgToolCalls - [double]$BaselineDigest.avgToolCalls), 1))).") + [void]$sb.AppendLine("- Skill activations: $BaselineVariant = $($BaselineDigest.totalSkillActivations), $SkillVariant = $($SkillDigest.totalSkillActivations).") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Evidence From Outcomes") + [void]$sb.AppendLine() + [void]$sb.AppendLine("- Average score: $BaselineVariant = $baselineScore%, $SkillVariant = $skillScore% (delta $scoreDelta points).") + [void]$sb.AppendLine("- Stimulus pass rate: $BaselineVariant = $baselinePassRate%, $SkillVariant = $skillPassRate% (delta $passRateDelta points).") + [void]$sb.AppendLine("- Total errors: $BaselineVariant = $($BaselineDigest.totalErrors), $SkillVariant = $($SkillDigest.totalErrors).") + [void]$sb.AppendLine("- Failed graders: $BaselineVariant = $($BaselineDigest.totalFailedGraders), $SkillVariant = $($SkillDigest.totalFailedGraders) (delta $([int]$SkillDigest.totalFailedGraders - [int]$BaselineDigest.totalFailedGraders)).") + [void]$sb.AppendLine("- Timeout-related grader failures: $BaselineVariant = $($BaselineDigest.totalTimeoutGraders), $SkillVariant = $($SkillDigest.totalTimeoutGraders) (delta $([int]$SkillDigest.totalTimeoutGraders - [int]$BaselineDigest.totalTimeoutGraders)).") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Efficiency Tradeoffs") + [void]$sb.AppendLine() + [void]$sb.AppendLine("- Tokens: $BaselineVariant = $([int]$baselineTokens), $SkillVariant = $([int]$skillTokens) (delta $tokenDeltaPct%).") + [void]$sb.AppendLine("- Wall time: $BaselineVariant = $baselineWall s, $SkillVariant = $skillWall s (delta $wallTimeDeltaPct%).") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Final Assessment") + [void]$sb.AppendLine() + [void]$sb.AppendLine("For $SkillName, the deterministic comparison indicates $overallEffect.") + if ([int]$SkillDigest.totalTimeoutGraders -gt [int]$BaselineDigest.totalTimeoutGraders) { + [void]$sb.AppendLine("Timeout-related grader failures increased in $SkillVariant; this may indicate the stimulus timeout threshold is too low for the executed workflow and should be reviewed.") + } + [void]$sb.AppendLine("This assessment is computed directly from trajectory digests to avoid narrative drift or missing-context artifacts.") + + return $sb.ToString().Trim() +} + +function Generate-SkillNarratives { + param( + [Parameter(Mandatory = $true)] + [string]$SkillName, + + [Parameter(Mandatory = $true)] + [string]$ExperimentOutputDir, + + [Parameter(Mandatory = $true)] + [string]$SkillResultsDir, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10)] + [int]$NarrativeMaxAttempts = 2, + + [Parameter(Mandatory = $false)] + [bool]$EnableNodeLoaderFallback = $true + ) + + if (-not (Test-Path $ExperimentOutputDir)) { + return $false + } + + New-Item -ItemType Directory -Path $SkillResultsDir -Force | Out-Null + + $variantDirs = Get-ChildItem -Path $ExperimentOutputDir -Directory | Where-Object { + Test-Path (Join-Path $_.FullName "results.jsonl") + } + + if ($variantDirs.Count -lt 2) { + Write-Warning "Expected at least 2 variants in $ExperimentOutputDir for $SkillName. Found $($variantDirs.Count)." + return $false + } + + $digests = @{} + $narratives = @{} + + foreach ($variantDir in $variantDirs) { + $resultsJsonlPath = Join-Path $variantDir.FullName "results.jsonl" + $digest = Get-VariantDigest -ResultsJsonlPath $resultsJsonlPath + if (-not $digest) { + continue + } + + $variantName = [string]$digest.variant + $digests[$variantName] = $digest + + $digestPath = Join-Path $SkillResultsDir ("trajectory-digest-" + $variantName + ".json") + $digest | ConvertTo-Json -Depth 12 | Set-Content -Path $digestPath -Encoding UTF8 + + $narrative = New-VariantNarrativeFromDigest -Digest $digest + + $narrative = Normalize-NarrativeText -Text $narrative + + $narrativePath = Join-Path $SkillResultsDir ("narrative-" + $variantName + ".md") + $narrative | Set-Content -Path $narrativePath -Encoding UTF8BOM + $narratives[$variantName] = $narrative + } + + if ($digests.Count -lt 2 -or $narratives.Count -lt 2) { + Write-Warning "Could not build both variant narratives for $SkillName" + return $false + } + + $variantNames = @($digests.Keys | Sort-Object) + $baselineVariant = $variantNames | Where-Object { $_ -match 'baseline' } | Select-Object -First 1 + $skillVariant = $variantNames | Where-Object { $_ -match 'skill' } | Select-Object -First 1 + + if (-not $baselineVariant) { + $baselineVariant = $variantNames[0] + } + if (-not $skillVariant) { + $skillVariant = $variantNames | Where-Object { $_ -ne $baselineVariant } | Select-Object -First 1 + } + + if (-not $skillVariant) { + Write-Warning "Unable to identify comparison pair for $SkillName" + return $false + } + + $baselineDigest = $digests[$baselineVariant] + $skillDigest = $digests[$skillVariant] + + $comparison = New-ComparisonNarrativeFromDigests -SkillName $SkillName -BaselineVariant $baselineVariant -SkillVariant $skillVariant -BaselineDigest $baselineDigest -SkillDigest $skillDigest + + $comparison = Normalize-NarrativeText -Text $comparison + + $comparisonPath = Join-Path $SkillResultsDir "narrative-comparison.md" + $comparison | Set-Content -Path $comparisonPath -Encoding UTF8BOM + + return $true +} + +$copilotCmd = Get-Command copilot -ErrorAction SilentlyContinue +if ($copilotCmd) { + Write-Information "Copilot CLI: $($copilotCmd.Source)" +} +else { + Write-Warning "copilot CLI not found. Narrative generation will be skipped." +} + +if ($AnalysisOnly) { + $analysisRoot = $ExperimentOutputDir + if ([string]::IsNullOrWhiteSpace($analysisRoot)) { + $analysisRoot = Get-LatestResultsDirectory -RootPath $ResultsRoot + } + + if (-not $analysisRoot) { + Write-Error "Analysis-only mode could not resolve a results directory. Provide -ExperimentOutputDir explicitly." + exit 1 + } + + $analysisExitCode = Generate-NarrativesOnly -AnalysisRoot $analysisRoot -SkillNamePattern $SkillPattern -NarrativeMaxAttempts $CopilotMaxAttempts -EnableNodeLoaderFallback (-not $NoNodeLoaderFallback) + exit $analysisExitCode +} + +# Validate vally is available +$vallyCmd = Get-Command vally -ErrorAction SilentlyContinue +if (-not $vallyCmd) { + Write-Error "vally command not found. Please install Vally to run experiments." + exit 1 +} + +Write-Information "Vally: $($vallyCmd.Source)" + +# Find all matching skill scenarios with experiments +$skillDirs = Get-ChildItem -Path $ScenariosRoot -Directory -Filter $SkillPattern | +Where-Object { Test-Path (Join-Path $_.FullName "vally" "skill_effectiveness_experiment.yaml") } | +Sort-Object Name + +Write-Information "Found $($skillDirs.Count) skills with experiments matching pattern: $SkillPattern" + +if ($skillDirs.Count -eq 0) { + Write-Warning "No skills found matching pattern: $SkillPattern" + exit 0 +} + +if ($DryRun) { + Write-Information "" + Write-Information "DRY RUN - Would execute the following experiments:" + foreach ($skillDir in $skillDirs) { + Write-Information " - $($skillDir.Name): $(Join-Path $skillDir.FullName 'vally' 'skill_effectiveness_experiment.yaml')" + } + exit 0 +} + +# Create results directory with timestamp +$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fffZ" +$experimentResultsDir = Join-Path $ResultsRoot $timestamp +New-Item -ItemType Directory -Path $experimentResultsDir -Force | Out-Null + +Write-Information "" +Write-Information "Results Directory: $experimentResultsDir" +Write-Information "Parallel Workers: $Workers" +Write-Information "" + +# Track results and timing +$results = @() +$completed = 0 +$failed = 0 +$totalSkills = $skillDirs.Count +$overallStartTime = Get-Date +$skillTimings = @() # Track timings for ETA calculation + +# Execute experiments +$scriptBlock = { + param($SkillDir, $ExperimentResultsDir, $ResultsRoot, $SkillIndex, $TotalSkills) + + $skillName = $SkillDir.Name + $vallyDir = Join-Path $SkillDir.FullName "vally" + try { + Write-Host "[$SkillIndex/$TotalSkills] ▶ Starting: $skillName" + + $startTime = Get-Date + $skillOutputRoot = Join-Path $ExperimentResultsDir $skillName + + # Run the experiment + # Note: vally experiment run expects to be run from the scenario directory + Push-Location $vallyDir + try { + $output = & vally experiment run skill_effectiveness_experiment.yaml --output-dir $skillOutputRoot 2>&1 + $exitCode = $LASTEXITCODE + } + finally { + Pop-Location + } + + $duration = (Get-Date) - $startTime + + # Check for success by looking at output and exit code + # Vally returns 0 on success, but also check for "passed" keyword in output + $outputString = $output -join "`n" + + # Strip ANSI color codes from output to allow proper regex matching + # vally uses ANSI codes for colored output which blocks metric extraction + $outputString = $outputString -replace "`e\[[0-9;]*m", '' + + # Parse experiment output directory locally so this works in Start-Job scope. + $experimentOutputDir = $null + $localOutputMatches = [regex]::Matches($outputString, 'Output:\s*(.+)') + if ($localOutputMatches.Count -gt 0) { + for ($m = $localOutputMatches.Count - 1; $m -ge 0; $m--) { + $candidatePath = $localOutputMatches[$m].Groups[1].Value.Trim() + if (Test-Path $candidatePath) { + $experimentOutputDir = $candidatePath + break + } + } + + if (-not $experimentOutputDir) { + $experimentOutputDir = $localOutputMatches[$localOutputMatches.Count - 1].Groups[1].Value.Trim() + } + } + + # Prefer deterministic per-variant metrics from JSONL results over parsing console text. + $baselineScore = $null + $skillScore = $null + $delta = $null + $baselineTokens = $null + $skillTokens = $null + $baselineTurns = $null + $skillTurns = $null + $baselineErrors = 0 + $skillErrors = 0 + $baselineFailedGraders = 0 + $skillFailedGraders = 0 + $baselineTimeoutGraders = 0 + $skillTimeoutGraders = 0 + $baselineEvalStatus = "UNKNOWN" + $skillEvalStatus = "UNKNOWN" + $succeeded = $false + + $variantStats = @{} + if ($experimentOutputDir -and (Test-Path $experimentOutputDir)) { + $variantDirs = Get-ChildItem -Path $experimentOutputDir -Directory -ErrorAction SilentlyContinue + foreach ($variantDir in $variantDirs) { + $resultsFile = Join-Path $variantDir.FullName "results.jsonl" + if (-not (Test-Path $resultsFile)) { + continue + } + + $scores = @() + $tokens = @() + $turns = @() + $errors = 0 + $failedGraders = 0 + $timeoutGraders = 0 + $trialCount = 0 + $passedCount = 0 + + foreach ($line in (Get-Content -Path $resultsFile -ErrorAction SilentlyContinue)) { + if ([string]::IsNullOrWhiteSpace($line)) { + continue + } + + try { + $record = $line | ConvertFrom-Json -ErrorAction Stop + } + catch { + continue + } + + if ($record.type -ne "trial-result") { + continue + } + + $trialCount++ + if ($record.gradeResult.passed) { + $passedCount++ + } + + if ($null -ne $record.gradeResult.score) { + $scores += ([double]$record.gradeResult.score * 100) + } + + if ($null -ne $record.trajectory.metrics.turnCount) { + $turns += [int]$record.trajectory.metrics.turnCount + } + + if ($null -ne $record.trajectory.metrics.tokenUsage.totalTokens) { + $tokens += [int]$record.trajectory.metrics.tokenUsage.totalTokens + } + + if ($null -ne $record.trajectory.metrics.errorCount) { + $errors += [int]$record.trajectory.metrics.errorCount + } + + if ($record.gradeResult -and $record.gradeResult.details) { + foreach ($detail in $record.gradeResult.details) { + if (-not [bool]$detail.passed) { + $failedGraders++ + $detailEvidence = if ($null -ne $detail.evidence) { [string]$detail.evidence } else { "" } + if ($detailEvidence -match '(?i)\btime(d)?\s*out\b|\btimeout\b') { + $timeoutGraders++ + } + } + } + } + } + + if ($trialCount -gt 0) { + $variantStats[$variantDir.Name] = @{ + AvgScore = if ($scores.Count -gt 0) { [double]($scores | Measure-Object -Average).Average } else { $null } + AvgTokens = if ($tokens.Count -gt 0) { [int](($tokens | Measure-Object -Average).Average) } else { $null } + AvgTurns = if ($turns.Count -gt 0) { [int](($turns | Measure-Object -Average).Average) } else { $null } + TotalErrors = $errors + FailedGraders = $failedGraders + TimeoutGraders = $timeoutGraders + TrialCount = $trialCount + PassedCount = $passedCount + } + } + } + } + + $baselineVariantName = $null + $skillVariantName = $null + if ($variantStats.Count -gt 0) { + $variantNames = @($variantStats.Keys | Sort-Object) + $baselineVariantName = $variantNames | Where-Object { $_ -match 'baseline' } | Select-Object -First 1 + $skillVariantName = $variantNames | Where-Object { $_ -match 'skill' } | Select-Object -First 1 + + if (-not $baselineVariantName) { + $baselineVariantName = $variantNames | Select-Object -First 1 + } + if (-not $skillVariantName) { + $skillVariantName = $variantNames | Where-Object { $_ -ne $baselineVariantName } | Select-Object -First 1 + } + } + + if ($baselineVariantName -and $variantStats.ContainsKey($baselineVariantName)) { + $baselineStats = $variantStats[$baselineVariantName] + $baselineScore = $baselineStats.AvgScore + $baselineTokens = $baselineStats.AvgTokens + $baselineTurns = $baselineStats.AvgTurns + $baselineErrors = $baselineStats.TotalErrors + $baselineFailedGraders = $baselineStats.FailedGraders + $baselineTimeoutGraders = $baselineStats.TimeoutGraders + if ($baselineStats.TrialCount -gt 0) { + $baselineEvalStatus = if ($baselineStats.PassedCount -eq $baselineStats.TrialCount) { "PASSED" } else { "FAILED" } + } + } + + if ($skillVariantName -and $variantStats.ContainsKey($skillVariantName)) { + $skillStats = $variantStats[$skillVariantName] + $skillScore = $skillStats.AvgScore + $skillTokens = $skillStats.AvgTokens + $skillTurns = $skillStats.AvgTurns + $skillErrors = $skillStats.TotalErrors + $skillFailedGraders = $skillStats.FailedGraders + $skillTimeoutGraders = $skillStats.TimeoutGraders + $succeeded = ($exitCode -eq 0) -and ($skillStats.TrialCount -gt 0) -and ($skillStats.PassedCount -eq $skillStats.TrialCount) + if ($skillStats.TrialCount -gt 0) { + $skillEvalStatus = if ($skillStats.PassedCount -eq $skillStats.TrialCount) { "PASSED" } else { "FAILED" } + } + } + else { + # Fallback for older outputs when per-variant JSONL is unavailable. + $succeeded = ($exitCode -eq 0) -and ($outputString -match "passed") + } + + if (($null -ne $baselineScore) -and ($null -ne $skillScore)) { + $delta = [double]$skillScore - [double]$baselineScore + } + + if ($succeeded) { + # Format impact indicator with efficiency metrics + $impactStr = "" + if ($delta -ne $null) { + $impactDirection = if ($delta -ge 0) { "↑" } else { "↓" } + $impactStr = " [baseline: $($baselineScore.ToString('F1'))% → skill: $($skillScore.ToString('F1'))% $impactDirection $($delta.ToString('+0.0;-0.0'))%]" + } + elseif ($skillScore) { + $impactStr = " [skill: $($skillScore.ToString('F1'))%]" + } + if ($skillEvalStatus -ne "UNKNOWN") { + $impactStr += " [skill eval: $skillEvalStatus]" + } + if ($skillFailedGraders -gt 0) { + $impactStr += " [failed graders: $skillFailedGraders" + if ($skillTimeoutGraders -gt 0) { + $impactStr += ", timeout-related: $skillTimeoutGraders" + } + $impactStr += "]" + } + + $durationText = "{0}h {1}m {2:F1}s" -f [int]$duration.TotalHours, $duration.Minutes, ($duration.Seconds + ($duration.Milliseconds / 1000)) + + Write-Host "[$SkillIndex/$TotalSkills] ✓ PASSED (run): $skillName ($durationText)$impactStr" -ForegroundColor Green + return @{ + Skill = $skillName + Status = "PASSED" + SkillEvalStatus = $skillEvalStatus + Duration = $duration.TotalSeconds + BaselineScore = $baselineScore + SkillScore = $skillScore + Delta = $delta + BaselineTokens = $baselineTokens + SkillTokens = $skillTokens + BaselineTurns = $baselineTurns + SkillTurns = $skillTurns + BaselineErrors = $baselineErrors + SkillErrors = $skillErrors + BaselineFailedGraders = $baselineFailedGraders + SkillFailedGraders = $skillFailedGraders + BaselineTimeoutGraders = $baselineTimeoutGraders + SkillTimeoutGraders = $skillTimeoutGraders + BaselineEvalStatus = $baselineEvalStatus + ExperimentDir = $experimentOutputDir + Output = $output + } + } + else { + Write-Host "[$SkillIndex/$TotalSkills] ✗ FAILED: $skillName (exit code: $exitCode)" -ForegroundColor Red + return @{ + Skill = $skillName + Status = "FAILED" + SkillEvalStatus = $skillEvalStatus + Duration = $duration.TotalSeconds + BaselineScore = $baselineScore + SkillScore = $skillScore + Delta = $delta + BaselineTokens = $baselineTokens + SkillTokens = $skillTokens + BaselineTurns = $baselineTurns + SkillTurns = $skillTurns + BaselineErrors = $baselineErrors + SkillErrors = $skillErrors + BaselineFailedGraders = $baselineFailedGraders + SkillFailedGraders = $skillFailedGraders + BaselineTimeoutGraders = $baselineTimeoutGraders + SkillTimeoutGraders = $skillTimeoutGraders + BaselineEvalStatus = $baselineEvalStatus + ExperimentDir = $experimentOutputDir + Output = $output + } + } + } + catch { + Write-Host "[$SkillIndex/$TotalSkills] ✗ Error running $skillName : $_" + return @{ + Skill = $skillName + Status = "ERROR" + Duration = 0 + Score = $null + Output = $_.Exception.Message + } + } +} + +# Run experiments with sequential or throttled-parallel execution +$skillIndex = 0 + +if ($Workers -eq 1) { + foreach ($skillDir in $skillDirs) { + $skillIndex++ + + $result = & $scriptBlock $skillDir $experimentResultsDir $ResultsRoot $skillIndex $totalSkills + $results += $result + $skillTimings += $result.Duration + + if ($result.Status -ne "PASSED") { + $failed++ + } + else { + $completed++ + } + + # Show ETA for sequential execution + if ($skillIndex -lt $totalSkills -and $skillTimings.Count -gt 0) { + $avgTime = ($skillTimings | Measure-Object -Average).Average + $remainingSkills = $totalSkills - $skillIndex + $estimatedSeconds = [int]($avgTime * $remainingSkills) + $estimatedTime = "{0:D2}:{1:D2}" -f [int]($estimatedSeconds / 60), [int]($estimatedSeconds % 60) + Write-Host " → Estimated time remaining: $estimatedTime" -ForegroundColor Cyan + } + } +} +else { + Write-Host "Launching experiments with worker limit: $Workers" -ForegroundColor Cyan + + $activeJobs = @() + $pendingSkills = [System.Collections.Queue]::new() + foreach ($skillDir in $skillDirs) { + [void]$pendingSkills.Enqueue($skillDir) + } + + function Start-NextExperimentJob { + param( + [Parameter(Mandatory = $true)] + [System.Collections.Queue]$Queue, + + [Parameter(Mandatory = $true)] + [int]$NextIndex, + + [Parameter(Mandatory = $true)] + [string]$ExpResultsDir, + + [Parameter(Mandatory = $true)] + [string]$ResRoot, + + [Parameter(Mandatory = $true)] + [int]$TotalCount, + + [Parameter(Mandatory = $true)] + [scriptblock]$RunnerScript + ) + + if ($Queue.Count -eq 0) { + return $null + } + + $nextSkillDir = $Queue.Dequeue() + $newJob = Start-Job -ScriptBlock $RunnerScript -ArgumentList $nextSkillDir, $ExpResultsDir, $ResRoot, $NextIndex, $TotalCount + + return @{ + Job = $newJob + Skill = $nextSkillDir.Name + SkillIndex = $NextIndex + } + } + + # Seed up to worker limit. + while ($activeJobs.Count -lt $Workers -and $pendingSkills.Count -gt 0) { + $skillIndex++ + $jobInfo = Start-NextExperimentJob -Queue $pendingSkills -NextIndex $skillIndex -ExpResultsDir $experimentResultsDir -ResRoot $ResultsRoot -TotalCount $totalSkills -RunnerScript $scriptBlock + if ($jobInfo) { + $activeJobs += $jobInfo + Write-Host " queued [$($jobInfo.SkillIndex)/$totalSkills] $($jobInfo.Skill) (active: $($activeJobs.Count)/$Workers)" -ForegroundColor DarkCyan + } + } + + # As jobs finish, consume output and backfill from queue. + while ($activeJobs.Count -gt 0) { + $jobObjects = @($activeJobs | ForEach-Object { $_.Job }) + $finishedJob = Wait-Job -Job $jobObjects -Any + if (-not $finishedJob) { + continue + } + + $jobInfo = $activeJobs | Where-Object { $_.Job.Id -eq $finishedJob.Id } | Select-Object -First 1 + $result = Receive-Job -Job $finishedJob -Wait -InformationAction SilentlyContinue + Remove-Job -Job $finishedJob + + $activeJobs = @($activeJobs | Where-Object { $_.Job.Id -ne $finishedJob.Id }) + + $results += $result + $skillTimings += $result.Duration + + if ($result.Status -ne "PASSED") { + $failed++ + } + else { + $completed++ + } + + $done = $completed + $failed + Write-Host " progress: $done/$totalSkills complete (active: $($activeJobs.Count), queued: $($pendingSkills.Count))" -ForegroundColor Cyan + + if ($pendingSkills.Count -gt 0) { + $skillIndex++ + $nextJobInfo = Start-NextExperimentJob -Queue $pendingSkills -NextIndex $skillIndex -ExpResultsDir $experimentResultsDir -ResRoot $ResultsRoot -TotalCount $totalSkills -RunnerScript $scriptBlock + if ($nextJobInfo) { + $activeJobs += $nextJobInfo + Write-Host " queued [$($nextJobInfo.SkillIndex)/$totalSkills] $($nextJobInfo.Skill) (active: $($activeJobs.Count)/$Workers)" -ForegroundColor DarkCyan + } + } + } +} + +Write-Information "" +Write-Information "Generating trajectory narratives and skill-impact comparisons..." + +$narrativeCount = 0 +$narrativeFailures = 0 + +foreach ($result in $results) { + if (-not $result.ExperimentDir) { + continue + } + + $skillNarrativeDir = Join-Path $experimentResultsDir $result.Skill + $generated = Generate-SkillNarratives -SkillName $result.Skill -ExperimentOutputDir $result.ExperimentDir -SkillResultsDir $skillNarrativeDir -NarrativeMaxAttempts $CopilotMaxAttempts -EnableNodeLoaderFallback (-not $NoNodeLoaderFallback) + + if ($generated) { + $narrativeCount++ + Write-Host " ✓ Narratives generated for $($result.Skill)" -ForegroundColor Green + } + else { + $narrativeFailures++ + Write-Host " ✗ Narrative generation skipped/failed for $($result.Skill)" -ForegroundColor Yellow + } +} + +if ($narrativeCount -gt 0 -or $narrativeFailures -gt 0) { + Write-Host "Narrative Summary: generated=$narrativeCount failed=$narrativeFailures" -ForegroundColor Cyan +} + +# Calculate overall timing +$overallDuration = (Get-Date) - $overallStartTime +$totalDurationStr = "{0}h {1}m {2}s" -f ` + [int]($overallDuration.TotalSeconds / 3600), ` + [int](($overallDuration.TotalSeconds % 3600) / 60), ` + [int]($overallDuration.TotalSeconds % 60) + +Write-Host "`n" + ("=" * 80) +Write-Host "Skill Effectiveness Experiments Summary" -ForegroundColor Green +Write-Host ("=" * 80) + +# Build results table with verdict and efficiency analysis +$resultsTable = @() +foreach ($result in $results) { + # Determine verdict based on pass/fail logic + $verdict = "" + $evalStatus = if ($result.SkillEvalStatus) { [string]$result.SkillEvalStatus } else { "UNKNOWN" } + $baselineEvalStatus = if ($result.BaselineEvalStatus) { [string]$result.BaselineEvalStatus } else { "UNKNOWN" } + $failureArm = "unknown" + + if ($result.Status -eq "FAILED") { + if ($baselineEvalStatus -eq "FAILED" -and $evalStatus -ne "FAILED") { + $failureArm = "baseline" + } + elseif ($evalStatus -eq "FAILED" -and $baselineEvalStatus -ne "FAILED") { + $failureArm = "skill" + } + elseif ($baselineEvalStatus -eq "FAILED" -and $evalStatus -eq "FAILED") { + $failureArm = "both" + } + else { + $failureArm = "run" + } + } + + if ($result.Status -eq "PASSED" -and $evalStatus -eq "PASSED") { + # Skill passed - check efficiency + if ($result.SkillTokens -and $result.BaselineTokens) { + $tokenDelta = [int](($result.SkillTokens - $result.BaselineTokens) / $result.BaselineTokens * 100) + if ($tokenDelta -lt -5) { + $verdict = "✓ Effective" + } + elseif ($tokenDelta -gt 5) { + $verdict = "✗ Inefficient" + } + else { + $verdict = "~ Neutral" + } + } + else { + $verdict = "✓ Passed" + } + } + elseif ($result.Status -eq "PASSED" -and $evalStatus -eq "FAILED") { + if (($result.SkillTimeoutGraders -as [int]) -gt 0) { + $verdict = "✗ Eval failed (timeout)" + } + else { + $verdict = "✗ Eval failed" + } + } + else { + $verdict = "✗ Failed" + } + + # Calculate efficiency deltas + $tokensDelta = "" + $turnsDelta = "" + + if ($result.SkillTokens -and $result.BaselineTokens) { + $pctChange = [int](($result.SkillTokens - $result.BaselineTokens) / $result.BaselineTokens * 100) + $dir = if ($pctChange -ge 0) { "↑" } else { "↓" } + $tokensDelta = "$dir $($pctChange.ToString('+0;-0'))%" + } + + if ($result.SkillTurns -and $result.BaselineTurns) { + $turnChange = $result.SkillTurns - $result.BaselineTurns + $dir = if ($turnChange -ge 0) { "↑" } else { "↓" } + $turnsDelta = "$dir $($turnChange.ToString('+0;-0'))" + } + + # Build status indicator + $statusIcon = if ($result.Status -eq "PASSED") { "RUN PASS" } else { "RUN FAIL" } + + # Build verdict display + $verdictDisplay = $verdict -replace "✓ ", "[GOOD] " -replace "✗ ", "[BAD] " -replace "~ ", "[NEUT] " + + $resultsTable += [PSCustomObject]@{ + Skill = $result.Skill + Status = $statusIcon + Eval = $evalStatus + 'Failure Arm' = if ($result.Status -eq "FAILED") { $failureArm } else { "none" } + Verdict = $verdictDisplay + 'Graders Failed' = if ($null -ne $result.SkillFailedGraders) { $result.SkillFailedGraders } else { "N/A" } + 'Graders Timeout' = if ($null -ne $result.SkillTimeoutGraders) { $result.SkillTimeoutGraders } else { "N/A" } + 'Tokens %' = if ($tokensDelta) { $tokensDelta } else { "N/A" } + 'Turns' = if ($turnsDelta) { $turnsDelta } else { "N/A" } + Errors = $result.SkillErrors + 'Duration' = Format-ElapsedDuration -Seconds $result.Duration + } +} + +# Display results table +$resultsTable | Format-Table -AutoSize + +Write-Host ("=" * 80) +Write-Host "Results Summary" -ForegroundColor Cyan +Write-Host (" Total Skills: $totalSkills") +Write-Host (" Passed: $completed") -ForegroundColor Green +if ($failed -gt 0) { + Write-Host (" Failed: $failed") -ForegroundColor Red +} +else { + Write-Host (" Failed: $failed") -ForegroundColor Green +} +Write-Host (" Total Duration: $totalDurationStr") -ForegroundColor Cyan +Write-Host (" Results Dir: $experimentResultsDir") -ForegroundColor Cyan +Write-Host ("=" * 80) +Write-Information "" + +if ($Compare) { + Write-Information "" + Write-Information "Generating comparison reports..." + + # Run compare-experiment.mjs for the experiment directory + $compareScript = Join-Path $PSScriptRoot "compare-experiment.mjs" + if (Test-Path $compareScript) { + & node $compareScript $experimentResultsDir + } + else { + Write-Warning "compare-experiment.mjs not found at: $compareScript" + } +} + +if ($failed -gt 0) { + exit 1 +} diff --git a/tests/scenarios/_shared/vally/azure-sdk-python-skill-evaluations.yaml b/tests/scenarios/_shared/vally/azure-sdk-python-skill-evaluations.yaml new file mode 100644 index 00000000..ec39cd44 --- /dev/null +++ b/tests/scenarios/_shared/vally/azure-sdk-python-skill-evaluations.yaml @@ -0,0 +1,43 @@ +name: azure-sdk-python-skill-evaluations + +evals: + - ../../agent-framework-azure-ai-py/vally/eval.yaml + - ../../azure-ai-contentsafety-py/vally/eval.yaml + - ../../azure-ai-contentunderstanding-py/vally/eval.yaml + - ../../azure-ai-language-conversations-py/vally/eval.yaml + - ../../azure-ai-ml-py/vally/eval.yaml + - ../../azure-ai-projects-py/vally/eval.yaml + - ../../azure-ai-textanalytics-py/vally/eval.yaml + - ../../azure-ai-transcription-py/vally/eval.yaml + - ../../azure-ai-translation-document-py/vally/eval.yaml + - ../../azure-ai-translation-text-py/vally/eval.yaml + - ../../azure-ai-vision-imageanalysis-py/vally/eval.yaml + - ../../azure-ai-voicelive-py/vally/eval.yaml + - ../../azure-appconfiguration-py/vally/eval.yaml + - ../../azure-containerregistry-py/vally/eval.yaml + - ../../azure-cosmos-db-py/vally/eval.yaml + - ../../azure-cosmos-py/vally/eval.yaml + - ../../azure-data-tables-py/vally/eval.yaml + - ../../azure-eventgrid-py/vally/eval.yaml + - ../../azure-eventhub-py/vally/eval.yaml + - ../../azure-identity-py/vally/eval.yaml + - ../../azure-keyvault-py/vally/eval.yaml + - ../../azure-messaging-webpubsubservice-py/vally/eval.yaml + - ../../azure-mgmt-apicenter-py/vally/eval.yaml + - ../../azure-mgmt-apimanagement-py/vally/eval.yaml + - ../../azure-mgmt-botservice-py/vally/eval.yaml + - ../../azure-mgmt-fabric-py/vally/eval.yaml + - ../../azure-monitor-ingestion-py/vally/eval.yaml + - ../../azure-monitor-opentelemetry-exporter-py/vally/eval.yaml + - ../../azure-monitor-opentelemetry-py/vally/eval.yaml + - ../../azure-monitor-query-py/vally/eval.yaml + - ../../azure-search-documents-py/vally/eval.yaml + - ../../azure-servicebus-py/vally/eval.yaml + - ../../azure-speech-to-text-rest-py/vally/eval.yaml + - ../../azure-storage-blob-py/vally/eval.yaml + - ../../azure-storage-file-datalake-py/vally/eval.yaml + - ../../azure-storage-file-share-py/vally/eval.yaml + - ../../azure-storage-queue-py/vally/eval.yaml + - ../../fastapi-router-py/vally/eval.yaml + - ../../m365-agents-py/vally/eval.yaml + - ../../pydantic-models-py/vally/eval.yaml diff --git a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts index df07f2e8..786fbb03 100644 --- a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts +++ b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts @@ -424,9 +424,8 @@ class CargoBuildTrajectoryGrader implements Grader { name: "rust-cargo-build-failure-check", description: "Rust-specific grader: checks trajectory for failed cargo build tool calls, extracts Rust compiler errors (E0XXX), and scales score by failure ratio", - behavior: { execution: "single", requiresWorkspace: false }, + behavior: { requiresWorkspace: false }, determinism: "static", - portability: "t3a-scenario", reference: "reference-free", temporalScope: "trajectory-level", costProfile: "free", diff --git a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json index 7caae95e..839e049e 100644 --- a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json +++ b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json @@ -10,7 +10,7 @@ "typecheck": "tsc --noEmit" }, "devDependencies": { - "@microsoft/vally": "^0.6.0", + "@microsoft/vally": "^0.7.0", "@types/node": "^20.0.0", "typescript": "^5.4.5" }, diff --git a/tests/scenarios/_shared/vally/tools/check-python-idiomatic.mjs b/tests/scenarios/_shared/vally/tools/check-python-idiomatic.mjs new file mode 100644 index 00000000..b2d4bd52 --- /dev/null +++ b/tests/scenarios/_shared/vally/tools/check-python-idiomatic.mjs @@ -0,0 +1,60 @@ +// Heuristic checks for non-idiomatic Python patterns. +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; + +const EXCLUDED_DIRS = new Set([ + ".git", + ".hg", + ".svn", + ".venv", + "venv", + "__pycache__", + "node_modules", + ".vally", +]); + +function collectPythonFiles(dir, acc) { + for (const entry of readdirSync(dir)) { + if (entry.startsWith(".") && entry !== ".vally") continue; + if (EXCLUDED_DIRS.has(entry)) continue; + const full = join(dir, entry); + const st = statSync(full); + if (st.isDirectory()) { + collectPythonFiles(full, acc); + } else if (entry.endsWith(".py")) { + acc.push(full); + } + } +} + +const files = []; +collectPythonFiles(process.cwd(), files); + +if (files.length === 0) { + console.error("No Python files found to evaluate for idiomatic checks."); + process.exit(1); +} + +const failures = []; +for (const file of files) { + const text = readFileSync(file, "utf-8"); + if (text.includes("\t")) { + failures.push(`${file}: contains tab indentation`); + } + if (/^\s*from\s+\S+\s+import\s+\*/m.test(text)) { + failures.push(`${file}: uses wildcard import`); + } + if (/^\s*except\s*:\s*$/m.test(text)) { + failures.push(`${file}: uses bare except`); + } +} + +if (failures.length > 0) { + console.error("Non-idiomatic Python patterns found:"); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log(`No non-idiomatic patterns detected in ${files.length} Python file(s).`); diff --git a/tests/scenarios/_shared/vally/tools/check-python-syntax.mjs b/tests/scenarios/_shared/vally/tools/check-python-syntax.mjs new file mode 100644 index 00000000..3542ffff --- /dev/null +++ b/tests/scenarios/_shared/vally/tools/check-python-syntax.mjs @@ -0,0 +1,100 @@ +// Validates syntax for generated Python files. +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import { spawnSync } from "node:child_process"; + +const EXCLUDED_DIRS = new Set([ + ".git", + ".hg", + ".svn", + ".venv", + "venv", + "__pycache__", + "node_modules", + ".vally", +]); + +// Read optional config file for file-level exclusions +let excludePatterns = []; +const configPath = process.argv[2]; +if (configPath) { + try { + const configContent = readFileSync(configPath, "utf-8"); + const config = JSON.parse(configContent); + excludePatterns = config.excludePatterns || []; + } catch (err) { + console.error(`Warning: Could not read config file "${configPath}":`, err.message); + } +} + +function matchGlobPattern(filePath, pattern) { + // Simple glob pattern matcher (supports * and **) + const regexPattern = pattern + .replace(/\./g, "\\.") + .replace(/\*\*/g, "___DOUBLE_STAR___") + .replace(/\*/g, "[^/]*") + .replace(/___DOUBLE_STAR___/g, ".*"); + const regex = new RegExp(`^${regexPattern}$`); + return regex.test(filePath); +} + +function shouldExcludeFile(filePath, baseDir) { + const relativePath = relative(baseDir, filePath).replace(/\\/g, "/"); + for (const pattern of excludePatterns) { + if (matchGlobPattern(relativePath, pattern)) { + return true; + } + } + return false; +} + +function collectPythonFiles(dir, acc, baseDir) { + for (const entry of readdirSync(dir)) { + if (entry.startsWith(".") && entry !== ".vally") continue; + if (EXCLUDED_DIRS.has(entry)) continue; + const full = join(dir, entry); + const st = statSync(full); + if (st.isDirectory()) { + collectPythonFiles(full, acc, baseDir); + } else if (entry.endsWith(".py")) { + if (!shouldExcludeFile(full, baseDir)) { + acc.push(full); + } + } + } +} + +const cwd = process.cwd(); +const files = []; +collectPythonFiles(cwd, files, cwd); + +if (files.length === 0) { + console.error("No Python files found to validate."); + process.exit(1); +} + +const failures = []; +for (const file of files) { + const result = spawnSync("python", ["-m", "py_compile", file], { + encoding: "utf-8", + }); + if (result.status !== 0) { + failures.push({ file, stderr: result.stderr?.trim() ?? "" }); + } +} + +if (failures.length > 0) { + console.error("Python syntax validation failed:"); + for (const failure of failures) { + console.error(`- ${failure.file}`); + if (failure.stderr) { + console.error(failure.stderr); + } + } + process.exit(1); +} + +const message = `Validated syntax for ${files.length} Python file(s).${ + excludePatterns.length > 0 ? ` (${excludePatterns.length} pattern(s) excluded)` : "" +}`; +console.log(message); diff --git a/tests/scenarios/agent-framework-azure-ai-py/scenarios.yaml b/tests/scenarios/agent-framework-azure-ai-py/scenarios.yaml index 055a05b5..12b3e0f1 100644 --- a/tests/scenarios/agent-framework-azure-ai-py/scenarios.yaml +++ b/tests/scenarios/agent-framework-azure-ai-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 @@ -18,7 +18,7 @@ scenarios: - "create_agent" - "run" forbidden_patterns: - - "AgentsClient" # Should use provider for basic example + - "AgentsClient" # Should use provider for basic example tags: - basic - provider @@ -26,7 +26,7 @@ scenarios: import asyncio from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -38,7 +38,7 @@ scenarios: ) result = await agent.run("Hello!") print(result.text) - + asyncio.run(main()) # Persistent agent and thread usage @@ -59,7 +59,7 @@ scenarios: from agent_framework import AgentThread from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -71,7 +71,7 @@ scenarios: print(result.text) print(thread.conversation_id) print(thread.service_thread_id) - + asyncio.run(main()) # Hosted code interpreter tool @@ -84,7 +84,7 @@ scenarios: - "HostedFileContent" - "tools=code_tool" forbidden_patterns: - - "code_interpreter" # Avoid string tool names + - "code_interpreter" # Avoid string tool names tags: - tools - code-interpreter @@ -93,7 +93,7 @@ scenarios: from agent_framework import HostedCodeInterpreterTool, HostedFileContent from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -109,7 +109,7 @@ scenarios: ) result = await agent.run("Summarize the file") print(result.text) - + asyncio.run(main()) # Hosted file search with vector store @@ -133,7 +133,7 @@ scenarios: from agent_framework.azure import AzureAIAgentsProvider from azure.ai.agents.aio import AgentsClient from azure.identity.aio import AzureCliCredential - + async def main(): endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] async with ( @@ -160,7 +160,7 @@ scenarios: ) result = await agent.run("What does the document say?") print(result.text) - + asyncio.run(main()) # Hosted web search tool @@ -180,7 +180,7 @@ scenarios: from agent_framework import HostedWebSearchTool from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): os.environ["BING_CONNECTION_ID"] = "your-bing-connection-id" async with ( @@ -197,7 +197,7 @@ scenarios: ) result = await agent.run("What are the latest AI updates?") print(result.text) - + asyncio.run(main()) # MCP Streamable HTTP tool @@ -210,7 +210,7 @@ scenarios: - "async with" - "tools=mcp_tool" forbidden_patterns: - - "HostedMCPTool" # Use streamable tool in this scenario + - "HostedMCPTool" # Use streamable tool in this scenario tags: - mcp - tools @@ -219,7 +219,7 @@ scenarios: from agent_framework import MCPStreamableHTTPTool from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -236,7 +236,7 @@ scenarios: ) result = await agent.run("What is Azure AI Foundry?") print(result.text) - + asyncio.run(main()) # Hosted MCP tool @@ -249,7 +249,7 @@ scenarios: - "approval_mode" - "allowed_tools" forbidden_patterns: - - "MCPStreamableHTTPTool" # Keep hosted MCP isolated + - "MCPStreamableHTTPTool" # Keep hosted MCP isolated tags: - mcp - tools @@ -258,7 +258,7 @@ scenarios: from agent_framework import HostedMCPTool from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -276,7 +276,7 @@ scenarios: ) result = await agent.run("How do I use Azure Functions?") print(result.text) - + asyncio.run(main()) # Streaming responses @@ -289,14 +289,14 @@ scenarios: - "async for" - "chunk.text" forbidden_patterns: - - "stream=True" # Use run_stream instead + - "stream=True" # Use run_stream instead tags: - streaming mock_response: | import asyncio from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -311,5 +311,5 @@ scenarios: if chunk.text: print(chunk.text, end="", flush=True) print() - + asyncio.run(main()) diff --git a/tests/scenarios/agent-framework-azure-ai-py/test-output.txt b/tests/scenarios/agent-framework-azure-ai-py/test-output.txt new file mode 100644 index 00000000..c07b0b91 --- /dev/null +++ b/tests/scenarios/agent-framework-azure-ai-py/test-output.txt @@ -0,0 +1 @@ +Error: Experiment config file not found: Q:\src\skills\tests\scenarios\agent-framework-azure-ai-py\skill_effectiveness_experiment.yaml diff --git a/tests/scenarios/agent-framework-azure-ai-py/vally/eval.yaml b/tests/scenarios/agent-framework-azure-ai-py/vally/eval.yaml new file mode 100644 index 00000000..9c9e09ff --- /dev/null +++ b/tests/scenarios/agent-framework-azure-ai-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: agent-framework-azure-ai-py-eval +description: Vally evaluation suite for agent-framework-azure-ai-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_provider_agent + prompt: |- + Create a basic Azure AI agent using AzureAIAgentsProvider. + Use async context managers, run a message, and print the response. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap agent initialization and all agent method calls in try/except blocks + * Handle: ServiceError, HttpResponseError, ClientAuthenticationError, generic Exception + * Example: try: run = client.agents.create_run(...) except ServiceError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: persistent_agent_thread + prompt: |- + Show how to retrieve an existing agent and continue a conversation + using a persistent thread. Include conversation ID handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap agent initialization and all agent method calls in try/except blocks + * Handle: ServiceError, HttpResponseError, ClientAuthenticationError, generic Exception + * Example: try: run = client.agents.create_run(...) except ServiceError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - agent-framework-azure-ai-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..fb91afc5 --- /dev/null +++ b/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: agent-framework-azure-ai-py-eval +description: Vally evaluation suite for agent-framework-azure-ai-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_provider_agent + prompt: |- + Create a basic Azure AI agent using AzureAIAgentsProvider. + Use async context managers, run a message, and print the response. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap agent initialization and all agent method calls in try/except blocks + * Handle: ServiceError, HttpResponseError, ClientAuthenticationError, generic Exception + * Example: try: run = client.agents.create_run(...) except ServiceError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: persistent_agent_thread + prompt: |- + Show how to retrieve an existing agent and continue a conversation + using a persistent thread. Include conversation ID handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap agent initialization and all agent method calls in try/except blocks + * Handle: ServiceError, HttpResponseError, ClientAuthenticationError, generic Exception + * Example: try: run = client.agents.create_run(...) except ServiceError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - agent-framework-azure-ai-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..2a2cb90e --- /dev/null +++ b/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: agent-framework-azure-ai-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-contentsafety-py/scenarios.yaml b/tests/scenarios/azure-ai-contentsafety-py/scenarios.yaml index c8e4515b..9c69fcda 100644 --- a/tests/scenarios/azure-ai-contentsafety-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-contentsafety-py/scenarios.yaml @@ -4,7 +4,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-contentsafety-py/vally/eval.yaml b/tests/scenarios/azure-ai-contentsafety-py/vally/eval.yaml new file mode 100644 index 00000000..5775dc0f --- /dev/null +++ b/tests/scenarios/azure-ai-contentsafety-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-ai-contentsafety-py-eval +description: Vally evaluation suite for azure-ai-contentsafety-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: text_analysis_api_key + prompt: |- + Analyze a piece of text with Azure AI Content Safety using API key + authentication. Include category severity checks for hate, self-harm, + sexual, and violence. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: text_analysis_entra_id + prompt: |- + Analyze text using Microsoft Entra ID authentication with + DefaultAzureCredential. Include analyze_text usage. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-contentsafety-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..6470ab32 --- /dev/null +++ b/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-contentsafety-py-eval +description: Vally evaluation suite for azure-ai-contentsafety-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: text_analysis_api_key + prompt: |- + Analyze a piece of text with Azure AI Content Safety using API key + authentication. Include category severity checks for hate, self-harm, + sexual, and violence. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: text_analysis_entra_id + prompt: |- + Analyze text using Microsoft Entra ID authentication with + DefaultAzureCredential. Include analyze_text usage. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-contentsafety-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..9eb863a1 --- /dev/null +++ b/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-contentsafety-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-contentunderstanding-py/scenarios.yaml b/tests/scenarios/azure-ai-contentunderstanding-py/scenarios.yaml index 2d4f9c92..561a65da 100644 --- a/tests/scenarios/azure-ai-contentunderstanding-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-contentunderstanding-py/scenarios.yaml @@ -4,7 +4,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-contentunderstanding-py/vally/eval.yaml b/tests/scenarios/azure-ai-contentunderstanding-py/vally/eval.yaml new file mode 100644 index 00000000..2cd04cc1 --- /dev/null +++ b/tests/scenarios/azure-ai-contentunderstanding-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-contentunderstanding-py-eval +description: Vally evaluation suite for azure-ai-contentunderstanding-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_authentication + prompt: |- + Create a basic Content Understanding client using DefaultAzureCredential. + Use the environment variable CONTENTUNDERSTANDING_ENDPOINT. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: document_extraction_url + prompt: |- + Analyze a PDF document URL using prebuilt-documentSearch and print the markdown. + Use begin_analyze with AnalyzeInput. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-contentunderstanding-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..2cd04cc1 --- /dev/null +++ b/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-contentunderstanding-py-eval +description: Vally evaluation suite for azure-ai-contentunderstanding-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_authentication + prompt: |- + Create a basic Content Understanding client using DefaultAzureCredential. + Use the environment variable CONTENTUNDERSTANDING_ENDPOINT. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: document_extraction_url + prompt: |- + Analyze a PDF document URL using prebuilt-documentSearch and print the markdown. + Use begin_analyze with AnalyzeInput. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-contentunderstanding-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..a97b7096 --- /dev/null +++ b/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-contentunderstanding-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-language-conversations-py/acceptance-criteria.md b/tests/scenarios/azure-ai-language-conversations-py/acceptance-criteria.md index 03fda1f5..31ea369d 100644 --- a/tests/scenarios/azure-ai-language-conversations-py/acceptance-criteria.md +++ b/tests/scenarios/azure-ai-language-conversations-py/acceptance-criteria.md @@ -1,16 +1,69 @@ -# Acceptance Criteria +# Acceptance Criteria: azure-ai-language-conversations-py ## Authentication and Setup -- [ ] MUST use `AzureKeyCredential` from `azure.core.credentials` for authentication. -- [ ] MUST initialize `ConversationAnalysisClient` correctly with an endpoint and credential. -- [ ] MUST NOT hardcode API keys or endpoints in the code examples. + +### ✅ Correct +```python +from azure.identity import DefaultAzureCredential +from azure.ai.language.conversations import ConversationAnalysisClient + +with ConversationAnalysisClient(endpoint, DefaultAzureCredential()) as client: + ... +``` + +### ✅ Correct: legacy key path for existing keyed deployments +```python +from azure.core.credentials import AzureKeyCredential +from azure.ai.language.conversations import ConversationAnalysisClient + +with ConversationAnalysisClient(endpoint, AzureKeyCredential(key)) as client: + ... +``` + +### ❌ Incorrect +```python +client = ConversationAnalysisClient(endpoint, credential) +# Missing context manager +``` ## Payload Construction -- [ ] MUST properly define the `task` dictionary with `"kind": "Conversation"`. -- [ ] MUST include a correctly structured `analysisInput` with `conversationItem` details (id, participantId, modality, text). -- [ ] MUST include required `parameters` (`projectName` and `deploymentName`). - -## API Usage and Extraction -- [ ] MUST demonstrate calling `analyze_conversation`. -- [ ] MUST show how to correctly parse the response dictionary to extract the `topIntent` or entities. -- [ ] MUST use a context manager (`with client:`) for the client instance. \ No newline at end of file + +### ✅ Correct +```python +task = { + "kind": "Conversation", + "analysisInput": { + "conversationItem": { + "participantId": "1", + "id": "1", + "modality": "text", + "text": query, + } + }, + "parameters": { + "projectName": project_name, + "deploymentName": deployment_name, + }, +} +``` + +### ❌ Incorrect +```python +task = { + "kind": "conversations", # Wrong kind + "parameters": {}, +} +``` + +## API Usage + +### ✅ Correct +```python +result = client.analyze_conversation(task=task) +print(result["result"]["prediction"]["topIntent"]) +``` + +### ❌ Incorrect +```python +result = client.analyze(task=task) # Wrong method name +``` diff --git a/tests/scenarios/azure-ai-language-conversations-py/scenarios.yaml b/tests/scenarios/azure-ai-language-conversations-py/scenarios.yaml new file mode 100644 index 00000000..2a396471 --- /dev/null +++ b/tests/scenarios/azure-ai-language-conversations-py/scenarios.yaml @@ -0,0 +1,102 @@ +config: + model: gpt-5.5 + max_tokens: 2000 + temperature: 0.3 + +scenarios: + - name: clu_basic_defaultazurecredential + prompt: | + Create a Python example that uses ConversationAnalysisClient with DefaultAzureCredential. + Build a valid task payload for Conversation analysis and print the top intent. + Use a context manager for the client. + expected_patterns: + - "DefaultAzureCredential" + - "ConversationAnalysisClient" + - "with ConversationAnalysisClient" + - "analyze_conversation" + - '"kind": "Conversation"' + - "projectName" + - "deploymentName" + - "topIntent" + forbidden_patterns: + - "client = ConversationAnalysisClient(" + - "analyze(" + tags: + - basic + - authentication + - payload + mock_response: | + import os + from azure.identity import DefaultAzureCredential + from azure.ai.language.conversations import ConversationAnalysisClient + + endpoint = os.environ["AZURE_CONVERSATIONS_ENDPOINT"] + project_name = os.environ["AZURE_CONVERSATIONS_PROJECT"] + deployment_name = os.environ["AZURE_CONVERSATIONS_DEPLOYMENT"] + + task = { + "kind": "Conversation", + "analysisInput": { + "conversationItem": { + "participantId": "1", + "id": "1", + "modality": "text", + "language": "en", + "text": "Book a flight to Seattle", + } + }, + "parameters": { + "projectName": project_name, + "deploymentName": deployment_name, + "verbose": True, + }, + } + + with ConversationAnalysisClient(endpoint, DefaultAzureCredential()) as client: + result = client.analyze_conversation(task=task) + print(result["result"]["prediction"]["topIntent"]) + + - name: clu_legacy_api_key_existing_deployments + prompt: | + Show the legacy API-key authentication path for an existing keyed deployment. + Use AzureKeyCredential with ConversationAnalysisClient in a context manager. + Mention this path is for existing deployments. + expected_patterns: + - "AzureKeyCredential" + - "ConversationAnalysisClient" + - "with ConversationAnalysisClient" + - "analyze_conversation" + forbidden_patterns: + - 'api_key="' + - "client = ConversationAnalysisClient(" + tags: + - legacy + - authentication + mock_response: | + import os + from azure.core.credentials import AzureKeyCredential + from azure.ai.language.conversations import ConversationAnalysisClient + + endpoint = os.environ["AZURE_CONVERSATIONS_ENDPOINT"] + key = os.environ["AZURE_CONVERSATIONS_KEY"] + + # Legacy path for existing keyed deployments; prefer DefaultAzureCredential for new code. + with ConversationAnalysisClient(endpoint, AzureKeyCredential(key)) as client: + result = client.analyze_conversation( + task={ + "kind": "Conversation", + "analysisInput": { + "conversationItem": { + "participantId": "1", + "id": "1", + "modality": "text", + "text": "What is my order status?", + } + }, + "parameters": { + "projectName": os.environ["AZURE_CONVERSATIONS_PROJECT"], + "deploymentName": os.environ["AZURE_CONVERSATIONS_DEPLOYMENT"], + }, + } + ) + print(result["result"]["prediction"]["topIntent"]) diff --git a/tests/scenarios/azure-ai-language-conversations-py/vally/eval.yaml b/tests/scenarios/azure-ai-language-conversations-py/vally/eval.yaml new file mode 100644 index 00000000..e143ffdf --- /dev/null +++ b/tests/scenarios/azure-ai-language-conversations-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-language-conversations-py-eval +description: Vally evaluation suite for azure-ai-language-conversations-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: clu_basic_defaultazurecredential + prompt: |- + Create a Python example that uses ConversationAnalysisClient with DefaultAzureCredential. + Build a valid task payload for Conversation analysis and print the top intent. + Use a context manager for the client. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: clu_legacy_api_key_existing_deployments + prompt: |- + Show the legacy API-key authentication path for an existing keyed deployment. + Use AzureKeyCredential with ConversationAnalysisClient in a context manager. + Mention this path is for existing deployments. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-language-conversations-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..39bd90f3 --- /dev/null +++ b/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-language-conversations-py-eval +description: Vally evaluation suite for azure-ai-language-conversations-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: clu_basic_defaultazurecredential + prompt: |- + Create a Python example that uses ConversationAnalysisClient with DefaultAzureCredential. + Build a valid task payload for Conversation analysis and print the top intent. + Use a context manager for the client. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: clu_legacy_api_key_existing_deployments + prompt: |- + Show the legacy API-key authentication path for an existing keyed deployment. + Use AzureKeyCredential with ConversationAnalysisClient in a context manager. + Mention this path is for existing deployments. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-language-conversations-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..de554732 --- /dev/null +++ b/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-language-conversations-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-ml-py/scenarios.yaml b/tests/scenarios/azure-ai-ml-py/scenarios.yaml index 9daa0b24..4ec9baa5 100644 --- a/tests/scenarios/azure-ai-ml-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-ml-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-ml-py/vally/eval.yaml b/tests/scenarios/azure-ai-ml-py/vally/eval.yaml new file mode 100644 index 00000000..60c158b3 --- /dev/null +++ b/tests/scenarios/azure-ai-ml-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-ai-ml-py-eval +description: Vally evaluation suite for azure-ai-ml-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: mlclient_creation_with_env_vars + prompt: |- + Create an MLClient instance using environment variables for authentication. + Include proper DefaultAzureCredential usage, and context manager pattern + for resource cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: create_and_list_workspaces + prompt: |- + Create a new Azure ML workspace with proper configuration, then list + all workspaces in the resource group. Include proper error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-ml-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-ml-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..235fa737 --- /dev/null +++ b/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-ml-py-eval +description: Vally evaluation suite for azure-ai-ml-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: mlclient_creation_with_env_vars + prompt: |- + Create an MLClient instance using environment variables for authentication. + Include proper DefaultAzureCredential usage, and context manager pattern + for resource cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: create_and_list_workspaces + prompt: |- + Create a new Azure ML workspace with proper configuration, then list + all workspaces in the resource group. Include proper error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-ml-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-ml-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..de02a843 --- /dev/null +++ b/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-ml-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-ml-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-projects-py/scenarios.yaml b/tests/scenarios/azure-ai-projects-py/scenarios.yaml index fd92a540..28f7d377 100644 --- a/tests/scenarios/azure-ai-projects-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-projects-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-projects-py/vally/eval.yaml b/tests/scenarios/azure-ai-projects-py/vally/eval.yaml new file mode 100644 index 00000000..d3975725 --- /dev/null +++ b/tests/scenarios/azure-ai-projects-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: azure-ai-projects-py-eval +description: Vally evaluation suite for azure-ai-projects-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: client_creation + prompt: |- + Create an AIProjectClient with proper authentication using DefaultAzureCredential. + Use environment variables for the endpoint and include a context manager pattern. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: versioned_agent_creation + prompt: |- + Create a versioned agent using create_version() with PromptAgentDefinition. + Include a version label and description. The agent should be named "customer-support-agent". + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-projects-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-projects-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..752ced17 --- /dev/null +++ b/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-projects-py-eval +description: Vally evaluation suite for azure-ai-projects-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: client_creation + prompt: |- + Create an AIProjectClient with proper authentication using DefaultAzureCredential. + Use environment variables for the endpoint and include a context manager pattern. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: versioned_agent_creation + prompt: |- + Create a versioned agent using create_version() with PromptAgentDefinition. + Include a version label and description. The agent should be named "customer-support-agent". + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-projects-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-projects-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..a758a0af --- /dev/null +++ b/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-projects-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-projects-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-textanalytics-py/scenarios.yaml b/tests/scenarios/azure-ai-textanalytics-py/scenarios.yaml index 089b9869..0b8f1fdc 100644 --- a/tests/scenarios/azure-ai-textanalytics-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-textanalytics-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-textanalytics-py/vally/eval.yaml b/tests/scenarios/azure-ai-textanalytics-py/vally/eval.yaml new file mode 100644 index 00000000..8371375c --- /dev/null +++ b/tests/scenarios/azure-ai-textanalytics-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-textanalytics-py-eval +description: Vally evaluation suite for azure-ai-textanalytics-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_sentiment_analysis + prompt: |- + Create a basic Azure Text Analytics example that analyzes sentiment + of multiple documents. Include proper authentication with DefaultAzureCredential, + error handling, and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: sentiment_with_opinion_mining + prompt: |- + Create an example that analyzes sentiment with opinion mining (aspect-based sentiment). + Extract both the target and assessments for each opinion. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-textanalytics-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..8371375c --- /dev/null +++ b/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-textanalytics-py-eval +description: Vally evaluation suite for azure-ai-textanalytics-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_sentiment_analysis + prompt: |- + Create a basic Azure Text Analytics example that analyzes sentiment + of multiple documents. Include proper authentication with DefaultAzureCredential, + error handling, and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: sentiment_with_opinion_mining + prompt: |- + Create an example that analyzes sentiment with opinion mining (aspect-based sentiment). + Extract both the target and assessments for each opinion. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-textanalytics-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..e1791f10 --- /dev/null +++ b/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-textanalytics-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-transcription-py/scenarios.yaml b/tests/scenarios/azure-ai-transcription-py/scenarios.yaml index b7c3005b..b6fb1878 100644 --- a/tests/scenarios/azure-ai-transcription-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-transcription-py/scenarios.yaml @@ -1,6 +1,6 @@ config: max_tokens: 2000 - model: gpt-4 + model: gpt-5.5 temperature: 0.3 scenarios: - expected_patterns: diff --git a/tests/scenarios/azure-ai-transcription-py/vally/eval.yaml b/tests/scenarios/azure-ai-transcription-py/vally/eval.yaml new file mode 100644 index 00000000..8754576a --- /dev/null +++ b/tests/scenarios/azure-ai-transcription-py/vally/eval.yaml @@ -0,0 +1,86 @@ +name: azure-ai-transcription-py-eval +description: Vally evaluation suite for azure-ai-transcription-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-ai-transcription-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-transcription-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..0a7b3f12 --- /dev/null +++ b/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,87 @@ +name: azure-ai-transcription-py-eval +description: Vally evaluation suite for azure-ai-transcription-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-ai-transcription-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-transcription-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..22878bab --- /dev/null +++ b/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-transcription-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-translation-document-py/scenarios.yaml b/tests/scenarios/azure-ai-translation-document-py/scenarios.yaml index f4f5a761..4a739c77 100644 --- a/tests/scenarios/azure-ai-translation-document-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-translation-document-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-translation-document-py/vally/eval.yaml b/tests/scenarios/azure-ai-translation-document-py/vally/eval.yaml new file mode 100644 index 00000000..5b324a05 --- /dev/null +++ b/tests/scenarios/azure-ai-translation-document-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-translation-document-py-eval +description: Vally evaluation suite for azure-ai-translation-document-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_batch_translation + prompt: |- + Create a script that submits a batch of documents for translation from a + source Azure Storage container to a target container. Show proper client + initialization and job submission. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: multiple_target_languages + prompt: |- + Create a script that translates a batch of documents to multiple languages + (Spanish, French, German) in a single translation job. Show how to specify + different target containers for each language. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-translation-document-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..fe92089b --- /dev/null +++ b/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-translation-document-py-eval +description: Vally evaluation suite for azure-ai-translation-document-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_batch_translation + prompt: |- + Create a script that submits a batch of documents for translation from a + source Azure Storage container to a target container. Show proper client + initialization and job submission. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: multiple_target_languages + prompt: |- + Create a script that translates a batch of documents to multiple languages + (Spanish, French, German) in a single translation job. Show how to specify + different target containers for each language. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-translation-document-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..0d5dc06d --- /dev/null +++ b/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-translation-document-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-translation-text-py/scenarios.yaml b/tests/scenarios/azure-ai-translation-text-py/scenarios.yaml index 9e4b96cb..ce31fcaa 100644 --- a/tests/scenarios/azure-ai-translation-text-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-translation-text-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-translation-text-py/vally/eval.yaml b/tests/scenarios/azure-ai-translation-text-py/vally/eval.yaml new file mode 100644 index 00000000..d9febe94 --- /dev/null +++ b/tests/scenarios/azure-ai-translation-text-py/vally/eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-translation-text-py-eval +description: Vally evaluation suite for azure-ai-translation-text-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_text_translation + prompt: |- + Create a basic example that translates English text to Spanish and French. + Include proper authentication with Entra ID credentials, client initialization, + and proper handling of the translation results. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: language_detection + prompt: |- + Create an example that detects the language of provided text using the + Azure AI Text Translation service. Show both automatic detection via + translate and explicit detection via the detect method. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-translation-text-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d9febe94 --- /dev/null +++ b/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-translation-text-py-eval +description: Vally evaluation suite for azure-ai-translation-text-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_text_translation + prompt: |- + Create a basic example that translates English text to Spanish and French. + Include proper authentication with Entra ID credentials, client initialization, + and proper handling of the translation results. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: language_detection + prompt: |- + Create an example that detects the language of provided text using the + Azure AI Text Translation service. Show both automatic detection via + translate and explicit detection via the detect method. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-translation-text-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..d263c7f5 --- /dev/null +++ b/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-translation-text-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-vision-imageanalysis-py/scenarios.yaml b/tests/scenarios/azure-ai-vision-imageanalysis-py/scenarios.yaml index 63f2290f..fa4fc082 100644 --- a/tests/scenarios/azure-ai-vision-imageanalysis-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-vision-imageanalysis-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/eval.yaml b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/eval.yaml new file mode 100644 index 00000000..b19ecaa0 --- /dev/null +++ b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-vision-imageanalysis-py-eval +description: Vally evaluation suite for azure-ai-vision-imageanalysis-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: image_captioning_from_url + prompt: |- + Create a basic example that analyzes an image from a URL to generate a caption. + Include proper authentication with DefaultAzureCredential, context manager for cleanup, + and extract the caption text with confidence score. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: ocr_text_extraction + prompt: |- + Create an example that extracts text from an image using OCR. + Include handling the READ visual feature to get text blocks, lines, and words. + Display text with confidence scores and bounding polygons. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-vision-imageanalysis-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..b19ecaa0 --- /dev/null +++ b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-vision-imageanalysis-py-eval +description: Vally evaluation suite for azure-ai-vision-imageanalysis-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: image_captioning_from_url + prompt: |- + Create a basic example that analyzes an image from a URL to generate a caption. + Include proper authentication with DefaultAzureCredential, context manager for cleanup, + and extract the caption text with confidence score. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: ocr_text_extraction + prompt: |- + Create an example that extracts text from an image using OCR. + Include handling the READ visual feature to get text blocks, lines, and words. + Display text with confidence scores and bounding polygons. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-vision-imageanalysis-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..e9ecc2b1 --- /dev/null +++ b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-vision-imageanalysis-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-voicelive-py/scenarios.yaml b/tests/scenarios/azure-ai-voicelive-py/scenarios.yaml index a7a623b4..fb1d5eab 100644 --- a/tests/scenarios/azure-ai-voicelive-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-voicelive-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-voicelive-py/vally/eval.yaml b/tests/scenarios/azure-ai-voicelive-py/vally/eval.yaml new file mode 100644 index 00000000..f967ec45 --- /dev/null +++ b/tests/scenarios/azure-ai-voicelive-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-ai-voicelive-py-eval +description: Vally evaluation suite for azure-ai-voicelive-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_session_setup + prompt: |- + Create a minimal async Voice Live connection that configures a session + using RequestSession with instructions, modalities, and Server VAD. + Use DefaultAzureCredential and update the session. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: api_key_authentication + prompt: |- + Connect to Voice Live using AzureKeyCredential and configure a session + with text+audio modalities. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-voicelive-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..c5befe75 --- /dev/null +++ b/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-voicelive-py-eval +description: Vally evaluation suite for azure-ai-voicelive-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_session_setup + prompt: |- + Create a minimal async Voice Live connection that configures a session + using RequestSession with instructions, modalities, and Server VAD. + Use DefaultAzureCredential and update the session. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: api_key_authentication + prompt: |- + Connect to Voice Live using AzureKeyCredential and configure a session + with text+audio modalities. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-voicelive-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..1fafb4dd --- /dev/null +++ b/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-voicelive-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-appconfiguration-py/scenarios.yaml b/tests/scenarios/azure-appconfiguration-py/scenarios.yaml index bfdd9e22..869ff46d 100644 --- a/tests/scenarios/azure-appconfiguration-py/scenarios.yaml +++ b/tests/scenarios/azure-appconfiguration-py/scenarios.yaml @@ -1,5 +1,5 @@ config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 scenarios: diff --git a/tests/scenarios/azure-appconfiguration-py/vally/eval.yaml b/tests/scenarios/azure-appconfiguration-py/vally/eval.yaml new file mode 100644 index 00000000..af458061 --- /dev/null +++ b/tests/scenarios/azure-appconfiguration-py/vally/eval.yaml @@ -0,0 +1,86 @@ +name: azure-appconfiguration-py-eval +description: Vally evaluation suite for azure-appconfiguration-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-appconfiguration-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-appconfiguration-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..92e6387f --- /dev/null +++ b/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,87 @@ +name: azure-appconfiguration-py-eval +description: Vally evaluation suite for azure-appconfiguration-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-appconfiguration-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-appconfiguration-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..993429ab --- /dev/null +++ b/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-appconfiguration-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-containerregistry-py/scenarios.yaml b/tests/scenarios/azure-containerregistry-py/scenarios.yaml index bab3a762..b27b0010 100644 --- a/tests/scenarios/azure-containerregistry-py/scenarios.yaml +++ b/tests/scenarios/azure-containerregistry-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-containerregistry-py/vally/eval.yaml b/tests/scenarios/azure-containerregistry-py/vally/eval.yaml new file mode 100644 index 00000000..e104966f --- /dev/null +++ b/tests/scenarios/azure-containerregistry-py/vally/eval.yaml @@ -0,0 +1,140 @@ +name: azure-containerregistry-py-eval +description: Vally evaluation suite for azure-containerregistry-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_repository_listing + prompt: |- + Create a basic example that lists all repositories in an Azure Container Registry + using DefaultAzureCredential and a context manager for resource cleanup. + Include the registry endpoint from environment variables. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: repository_properties_management + prompt: |- + Write code to get repository properties, update the writeEnabled property to lock + a production repository from being modified, and then display the updated properties. + Use context manager for resource cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-containerregistry-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-containerregistry-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..e104966f --- /dev/null +++ b/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-containerregistry-py-eval +description: Vally evaluation suite for azure-containerregistry-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_repository_listing + prompt: |- + Create a basic example that lists all repositories in an Azure Container Registry + using DefaultAzureCredential and a context manager for resource cleanup. + Include the registry endpoint from environment variables. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: repository_properties_management + prompt: |- + Write code to get repository properties, update the writeEnabled property to lock + a production repository from being modified, and then display the updated properties. + Use context manager for resource cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-containerregistry-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-containerregistry-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..d33a5118 --- /dev/null +++ b/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-containerregistry-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-containerregistry-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-cosmos-db-py/scenarios.yaml b/tests/scenarios/azure-cosmos-db-py/scenarios.yaml index c7077927..57b1f846 100644 --- a/tests/scenarios/azure-cosmos-db-py/scenarios.yaml +++ b/tests/scenarios/azure-cosmos-db-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 @@ -28,10 +28,10 @@ scenarios: import os from azure.cosmos import CosmosClient from azure.identity import DefaultAzureCredential - + def _is_emulator_endpoint(endpoint: str) -> bool: return "localhost" in endpoint.lower() or "127.0.0.1" in endpoint - + def create_client(endpoint: str, cosmos_key: str) -> CosmosClient: if _is_emulator_endpoint(endpoint): return CosmosClient( @@ -43,7 +43,7 @@ scenarios: url=endpoint, credential=DefaultAzureCredential(), ) - + client = create_client( endpoint=os.environ["COSMOS_ENDPOINT"], cosmos_key=os.environ.get("COSMOS_KEY", ""), @@ -59,30 +59,98 @@ scenarios: - "_init_attempted" - "get_container" - "container.read()" - forbidden_patterns: - - "CosmosClient(" # client should be created in helper, not per call + - "_create_client" + # forbidden_patterns: + # - "CosmosClient(" # client should be created in helper, not per call tags: - client - singleton mock_response: | + import logging + from threading import Lock from typing import Optional - from azure.cosmos import ContainerProxy - from app.db.cosmos import create_client - from app.core.config import settings - + + from azure.cosmos import ContainerProxy, CosmosClient + from azure.identity import DefaultAzureCredential + + from app.config import settings + + logger = logging.getLogger(__name__) + + _cosmos_client: Optional[CosmosClient] = None _cosmos_container: Optional[ContainerProxy] = None - _init_attempted: bool = False - + _init_attempted = False + _init_lock = Lock() + + + def _is_emulator_endpoint(endpoint: str) -> bool: + return \"localhost\" in endpoint.lower() or \"127.0.0.1\" in endpoint + + + def _create_client() -> CosmosClient: + if _is_emulator_endpoint(settings.cosmos_endpoint): + return CosmosClient( + url=settings.cosmos_endpoint, + credential=settings.cosmos_key, + connection_verify=False, + ) + + return CosmosClient( + url=settings.cosmos_endpoint, + credential=DefaultAzureCredential(), + ) + + def get_container() -> Optional[ContainerProxy]: - global _cosmos_container, _init_attempted + global _cosmos_client, _cosmos_container, _init_attempted + if _init_attempted: return _cosmos_container - _init_attempted = True - client = create_client(settings.cosmos_endpoint, settings.cosmos_key) - database = client.get_database_client(settings.cosmos_database_name) - _cosmos_container = database.get_container_client(settings.cosmos_container_id) - _cosmos_container.read() - return _cosmos_container + + with _init_lock: + if _init_attempted: + return _cosmos_container + + _init_attempted = True + + try: + _cosmos_client = _create_client() + database = _cosmos_client.get_database_client(settings.cosmos_database_name) + _cosmos_container = database.get_container_client(settings.cosmos_container_id) + _cosmos_container.read() + + logger.info( + \"Cosmos DB connected\", + extra={ + \"database\": settings.cosmos_database_name, + \"container\": settings.cosmos_container_id, + \"endpoint\": settings.cosmos_endpoint[:30] + \"...\", + }, + ) + except Exception as exc: + logger.error( + \"Cosmos DB connection failed\", + extra={ + \"error_type\": type(exc).__name__, + \"error_message\": str(exc), + \"endpoint\": settings.cosmos_endpoint[:30] + \"...\", + }, + exc_info=True, + ) + _cosmos_container = None + + return _cosmos_container + + + def reset_connection() -> None: + global _cosmos_client, _cosmos_container, _init_attempted + + if _cosmos_client is not None: + _cosmos_client.close() + + _cosmos_client = None + _cosmos_container = None + _init_attempted = False # Service Layer CRUD - name: service_layer_crud @@ -104,11 +172,11 @@ scenarios: from typing import Optional from app.db.cosmos import get_container, get_document, upsert_document, delete_document, query_documents from app.models.project import Project, ProjectCreate, ProjectUpdate, ProjectInDB - + class ProjectService: def _use_cosmos(self) -> bool: return get_container() is not None - + def _doc_to_model_in_db(self, doc: dict) -> ProjectInDB: return ProjectInDB( id=doc["id"], @@ -116,7 +184,7 @@ scenarios: workspace_id=doc["workspaceId"], doc_type=doc.get("docType", "project"), ) - + def _model_in_db_to_doc(self, model: ProjectInDB) -> dict: return { "id": model.id, @@ -124,10 +192,10 @@ scenarios: "workspaceId": model.workspace_id, "docType": model.doc_type, } - + def _model_in_db_to_model(self, model: ProjectInDB) -> Project: return Project(id=model.id, name=model.name, workspace_id=model.workspace_id) - + async def create(self, data: ProjectCreate, author_id: str) -> Project: if not self._use_cosmos(): raise RuntimeError("Database unavailable") @@ -139,7 +207,7 @@ scenarios: ) await upsert_document(self._model_in_db_to_doc(model), partition_key=data.workspace_id) return self._model_in_db_to_model(model) - + async def get_by_id(self, project_id: str, workspace_id: str) -> Optional[Project]: if not self._use_cosmos(): return None @@ -147,7 +215,7 @@ scenarios: if doc is None: return None return self._model_in_db_to_model(self._doc_to_model_in_db(doc)) - + async def update(self, project_id: str, workspace_id: str, data: ProjectUpdate) -> Optional[Project]: if not self._use_cosmos(): return None @@ -161,18 +229,18 @@ scenarios: setattr(model, field, value) await upsert_document(self._model_in_db_to_doc(model), partition_key=workspace_id) return self._model_in_db_to_model(model) - + async def delete(self, project_id: str, workspace_id: str) -> bool: if not self._use_cosmos(): return False return await delete_document(project_id, partition_key=workspace_id) - + async def list_by_workspace(self, workspace_id: str) -> list[Project]: if not self._use_cosmos(): return [] docs = await query_documents(doc_type="project", partition_key=workspace_id) return [self._model_in_db_to_model(self._doc_to_model_in_db(doc)) for doc in docs] - + project_service = ProjectService() # Parameterized Query @@ -185,14 +253,14 @@ scenarios: - "@slug" - 'parameters=[{"name": "@slug"' forbidden_patterns: - - "f\"SELECT" - - "%s" # string formatting in query + - 'f"SELECT' + - "%s" # string formatting in query tags: - queries - security mock_response: | from app.db.cosmos import query_documents - + async def get_by_slug(slug: str, workspace_id: str): docs = await query_documents( doc_type="project", @@ -212,15 +280,15 @@ scenarios: - "partition_key=workspace_id" - "GLOBAL_PARTITION" forbidden_patterns: - - "partition_key=None" # should use a known partition when possible + - "partition_key=None" # should use a known partition when possible tags: - partitioning mock_response: | GLOBAL_PARTITION = "global" - + async def get_project(project_id: str, workspace_id: str): return await get_document(project_id, partition_key=workspace_id) - + async def get_user_by_email(email: str): return await query_documents( doc_type="user", @@ -245,7 +313,7 @@ scenarios: mock_response: | from starlette.concurrency import run_in_threadpool from app.db.cosmos import get_container - + async def list_all_users(): container = get_container() query = "SELECT * FROM c WHERE c.docType = @docType" @@ -268,7 +336,7 @@ scenarios: - "mocker.patch" - "mock_cosmos_container" forbidden_patterns: - - "CosmosClient(" # unit tests should not initialize real clients + - "CosmosClient(" # unit tests should not initialize real clients tags: - tests - tdd @@ -276,13 +344,13 @@ scenarios: import pytest from unittest.mock import MagicMock from app.services.project_service import project_service - + @pytest.fixture def mock_cosmos_container(mocker): container = MagicMock() mocker.patch("app.db.cosmos.get_container", return_value=container) return container - + @pytest.mark.asyncio async def test_get_by_id_returns_project(mock_cosmos_container): mock_cosmos_container.read_item.return_value = {"id": "123", "name": "Test", "workspaceId": "ws-1"} @@ -307,9 +375,9 @@ scenarios: mock_response: | from fastapi import APIRouter from app.services.project_service import project_service - + router = APIRouter() - + @router.get("/workspaces/{workspace_id}/projects/{project_id}") async def get_project(project_id: str, workspace_id: str): return await project_service.get_by_id(project_id, workspace_id) diff --git a/tests/scenarios/azure-cosmos-db-py/vally/eval.yaml b/tests/scenarios/azure-cosmos-db-py/vally/eval.yaml new file mode 100644 index 00000000..0804934e --- /dev/null +++ b/tests/scenarios/azure-cosmos-db-py/vally/eval.yaml @@ -0,0 +1,141 @@ +name: azure-cosmos-db-py-eval +description: Vally evaluation suite for azure-cosmos-db-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: dual_auth_client_setup + prompt: |- + Create a Cosmos DB client setup with dual authentication. + Use DefaultAzureCredential in Azure and key-based auth for emulator. + Include emulator detection and SSL verification disabled for emulator. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all CRUD operations in try/except blocks + * Handle: ClientAuthenticationError, CosmosHttpResponseError, AzureError + * Example: try: item = container.upsert_item(body) except CosmosHttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Includes explicit try/except exception handling. + - Provides complete executable Python code with clear control flow. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when: + 1. The code includes try/except blocks (explicit exception handling required) + 2. Uses Pythonic naming and clear control flow + 3. Avoids wildcard imports + 4. Uses context managers or lifecycle-safe patterns where relevant + 5. Uses proper error handling (no bare except clauses) + Fail immediately if no Python code is generated or if there are no try/except blocks. + scoring: scale_1_5 + - name: singleton_container_initialization + prompt: |- + Show a singleton Cosmos container initialization with a cached container + and a safe one-time initialization flag. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all CRUD operations in try/except blocks + * Handle: ClientAuthenticationError, CosmosHttpResponseError, AzureError + * Example: try: item = container.upsert_item(body) except CosmosHttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Includes explicit try/except exception handling. + - Provides complete executable Python code with clear control flow. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when: + 1. The code includes try/except blocks (explicit exception handling required) + 2. Uses Pythonic naming and clear control flow + 3. Avoids wildcard imports + 4. Uses context managers or lifecycle-safe patterns where relevant + 5. Uses proper error handling (no bare except clauses) + Fail immediately if no Python code is generated or if there are no try/except blocks. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-cosmos-db-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..0804934e --- /dev/null +++ b/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,141 @@ +name: azure-cosmos-db-py-eval +description: Vally evaluation suite for azure-cosmos-db-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: dual_auth_client_setup + prompt: |- + Create a Cosmos DB client setup with dual authentication. + Use DefaultAzureCredential in Azure and key-based auth for emulator. + Include emulator detection and SSL verification disabled for emulator. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all CRUD operations in try/except blocks + * Handle: ClientAuthenticationError, CosmosHttpResponseError, AzureError + * Example: try: item = container.upsert_item(body) except CosmosHttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Includes explicit try/except exception handling. + - Provides complete executable Python code with clear control flow. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when: + 1. The code includes try/except blocks (explicit exception handling required) + 2. Uses Pythonic naming and clear control flow + 3. Avoids wildcard imports + 4. Uses context managers or lifecycle-safe patterns where relevant + 5. Uses proper error handling (no bare except clauses) + Fail immediately if no Python code is generated or if there are no try/except blocks. + scoring: scale_1_5 + - name: singleton_container_initialization + prompt: |- + Show a singleton Cosmos container initialization with a cached container + and a safe one-time initialization flag. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all CRUD operations in try/except blocks + * Handle: ClientAuthenticationError, CosmosHttpResponseError, AzureError + * Example: try: item = container.upsert_item(body) except CosmosHttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Includes explicit try/except exception handling. + - Provides complete executable Python code with clear control flow. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when: + 1. The code includes try/except blocks (explicit exception handling required) + 2. Uses Pythonic naming and clear control flow + 3. Avoids wildcard imports + 4. Uses context managers or lifecycle-safe patterns where relevant + 5. Uses proper error handling (no bare except clauses) + Fail immediately if no Python code is generated or if there are no try/except blocks. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-cosmos-db-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..43719fc0 --- /dev/null +++ b/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-cosmos-db-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-cosmos-py/scenarios.yaml b/tests/scenarios/azure-cosmos-py/scenarios.yaml index dd6000dc..8c64fa1b 100644 --- a/tests/scenarios/azure-cosmos-py/scenarios.yaml +++ b/tests/scenarios/azure-cosmos-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-cosmos-py/vally/eval.yaml b/tests/scenarios/azure-cosmos-py/vally/eval.yaml new file mode 100644 index 00000000..98b0c3c9 --- /dev/null +++ b/tests/scenarios/azure-cosmos-py/vally/eval.yaml @@ -0,0 +1,149 @@ +name: azure-cosmos-py-eval +description: Vally evaluation suite for azure-cosmos-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_aad_client + prompt: |- + Create a basic Cosmos DB NoSQL example using DefaultAzureCredential. + Get a database and container, upsert an item, and read it back. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Ensure ALL Python syntax is valid and correct: + * Use proper bracket matching: {}, [], () + * Do not use placeholder variables or template syntax + - All .py files must pass Python syntax validation: `python -m py_compile ` + - CRITICAL: Include explicit exception handling: + * Wrap credential acquisition in try/except + * Wrap client initialization in try/except + * Wrap all CRUD operations (upsert, read) in try/except blocks + * Handle specific exceptions: ClientAuthenticationError, CosmosHttpResponseError, exceptions.AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: handle_auth_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: connection_string_client + prompt: |- + Create a CosmosClient using a connection string and retrieve + an existing database. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Ensure ALL Python syntax is valid and correct: + * Use proper bracket matching: {}, [], () + * Do not use placeholder variables or template syntax + - All .py files must pass Python syntax validation: `python -m py_compile ` + - CRITICAL: Include explicit exception handling: + * Wrap client initialization in try/except + * Wrap database retrieval in try/except blocks + * Handle specific exceptions: ClientAuthenticationError, CosmosHttpResponseError, exceptions.AzureError + * Example: try: client = CosmosClient.from_connection_string(conn) except ClientAuthenticationError as e: handle_auth_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-cosmos-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..98b0c3c9 --- /dev/null +++ b/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,149 @@ +name: azure-cosmos-py-eval +description: Vally evaluation suite for azure-cosmos-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_aad_client + prompt: |- + Create a basic Cosmos DB NoSQL example using DefaultAzureCredential. + Get a database and container, upsert an item, and read it back. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Ensure ALL Python syntax is valid and correct: + * Use proper bracket matching: {}, [], () + * Do not use placeholder variables or template syntax + - All .py files must pass Python syntax validation: `python -m py_compile ` + - CRITICAL: Include explicit exception handling: + * Wrap credential acquisition in try/except + * Wrap client initialization in try/except + * Wrap all CRUD operations (upsert, read) in try/except blocks + * Handle specific exceptions: ClientAuthenticationError, CosmosHttpResponseError, exceptions.AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: handle_auth_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: connection_string_client + prompt: |- + Create a CosmosClient using a connection string and retrieve + an existing database. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Ensure ALL Python syntax is valid and correct: + * Use proper bracket matching: {}, [], () + * Do not use placeholder variables or template syntax + - All .py files must pass Python syntax validation: `python -m py_compile ` + - CRITICAL: Include explicit exception handling: + * Wrap client initialization in try/except + * Wrap database retrieval in try/except blocks + * Handle specific exceptions: ClientAuthenticationError, CosmosHttpResponseError, exceptions.AzureError + * Example: try: client = CosmosClient.from_connection_string(conn) except ClientAuthenticationError as e: handle_auth_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-cosmos-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..a88954b6 --- /dev/null +++ b/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-cosmos-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml index d0ee97e9..7ffa851e 100644 --- a/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml +++ b/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-cosmos-crud-operations @@ -62,7 +62,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper Result-based error propagation." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-cosmos-rust/vally/eval.yaml b/tests/scenarios/azure-cosmos-rust/vally/eval.yaml index 097bc098..15380df1 100644 --- a/tests/scenarios/azure-cosmos-rust/vally/eval.yaml +++ b/tests/scenarios/azure-cosmos-rust/vally/eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-cosmos-crud-operations @@ -62,7 +62,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper Result-based error propagation." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -84,7 +84,7 @@ stimuli: - type: prompt name: "Cosmos DB CRUD completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - CosmosClient creation with builder and ProximityTo routing - Database and container client navigation @@ -166,7 +166,7 @@ stimuli: - name: CosmosClient Builder Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CosmosClient created via CosmosClient::builder() .build(account, RoutingStrategy::ProximityTo(region)).await? @@ -175,7 +175,7 @@ stimuli: - name: CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Create: container.create_item(partition_key, id, item, None).await? Read: container.read_item(partition_key, id, None).await? @@ -185,7 +185,7 @@ stimuli: - name: Partition Key Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Partition key is required parameter in all item operations Partition key value passed as string to operations @@ -194,7 +194,7 @@ stimuli: - name: Patch Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Patch operations use PatchInstructions::from(vec![...]) Operations created with PatchOperation::set(path, value) @@ -203,7 +203,7 @@ stimuli: - name: Response Conversion and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Responses converted with into_model() for deserialization Error handling with match and error extraction diff --git a/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml index da3af2ca..d39c0196 100644 --- a/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-cosmos-crud-operations @@ -62,7 +62,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper Result-based error propagation." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -90,7 +90,7 @@ stimuli: - type: prompt name: "Cosmos DB CRUD completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - CosmosClient creation with builder and ProximityTo routing - Database and container client navigation @@ -149,7 +149,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 2m - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -172,7 +172,7 @@ stimuli: - name: CosmosClient Builder Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CosmosClient created via CosmosClient::builder() .build(account, RoutingStrategy::ProximityTo(region)).await? @@ -181,7 +181,7 @@ stimuli: - name: CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Create: container.create_item(partition_key, id, item, None).await? Read: container.read_item(partition_key, id, None).await? @@ -191,7 +191,7 @@ stimuli: - name: Partition Key Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Partition key is required parameter in all item operations Partition key value passed as string to operations @@ -200,7 +200,7 @@ stimuli: - name: Patch Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Patch operations use PatchInstructions::from(vec![...]) Operations created with PatchOperation::set(path, value) @@ -209,7 +209,7 @@ stimuli: - name: Response Conversion and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Responses converted with into_model() for deserialization Error handling with match and error extraction diff --git a/tests/scenarios/azure-data-tables-py/scenarios.yaml b/tests/scenarios/azure-data-tables-py/scenarios.yaml index a5d105bc..ab1ef824 100644 --- a/tests/scenarios/azure-data-tables-py/scenarios.yaml +++ b/tests/scenarios/azure-data-tables-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-data-tables-py/vally/eval.yaml b/tests/scenarios/azure-data-tables-py/vally/eval.yaml new file mode 100644 index 00000000..5737fda8 --- /dev/null +++ b/tests/scenarios/azure-data-tables-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-data-tables-py-eval +description: Vally evaluation suite for azure-data-tables-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_table_service_client + prompt: |- + Create a TableServiceClient using DefaultAzureCredential, create a table + if it doesn't exist, and get a TableClient for that table. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: table_client_entity_crud + prompt: |- + Create a TableClient and perform entity CRUD operations using + PartitionKey/RowKey, including update with UpdateMode. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-data-tables-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-data-tables-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..5737fda8 --- /dev/null +++ b/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-data-tables-py-eval +description: Vally evaluation suite for azure-data-tables-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_table_service_client + prompt: |- + Create a TableServiceClient using DefaultAzureCredential, create a table + if it doesn't exist, and get a TableClient for that table. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: table_client_entity_crud + prompt: |- + Create a TableClient and perform entity CRUD operations using + PartitionKey/RowKey, including update with UpdateMode. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-data-tables-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-data-tables-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..8d345bb7 --- /dev/null +++ b/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-data-tables-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-data-tables-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-eventgrid-py/scenarios.yaml b/tests/scenarios/azure-eventgrid-py/scenarios.yaml index 3477b107..0d60917a 100644 --- a/tests/scenarios/azure-eventgrid-py/scenarios.yaml +++ b/tests/scenarios/azure-eventgrid-py/scenarios.yaml @@ -7,7 +7,7 @@ leakspeed: "" description: "" config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-eventgrid-py/vally/eval.yaml b/tests/scenarios/azure-eventgrid-py/vally/eval.yaml new file mode 100644 index 00000000..6c5a517c --- /dev/null +++ b/tests/scenarios/azure-eventgrid-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-eventgrid-py-eval +description: Vally evaluation suite for azure-eventgrid-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_cloudevent_publish + prompt: |- + Publish a single CloudEvent to an Event Grid custom topic. + Use DefaultAzureCredential, environment variables, and proper imports. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: batch_cloudevent_publish + prompt: |- + Send a batch of CloudEvents to Event Grid using a list. + Use DefaultAzureCredential and a topic endpoint from env. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-eventgrid-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventgrid-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..6c5a517c --- /dev/null +++ b/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-eventgrid-py-eval +description: Vally evaluation suite for azure-eventgrid-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_cloudevent_publish + prompt: |- + Publish a single CloudEvent to an Event Grid custom topic. + Use DefaultAzureCredential, environment variables, and proper imports. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: batch_cloudevent_publish + prompt: |- + Send a batch of CloudEvents to Event Grid using a list. + Use DefaultAzureCredential and a topic endpoint from env. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-eventgrid-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventgrid-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..0634c1b3 --- /dev/null +++ b/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-eventgrid-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventgrid-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-eventhub-py/scenarios.yaml b/tests/scenarios/azure-eventhub-py/scenarios.yaml index d85da2a3..c375b5eb 100644 --- a/tests/scenarios/azure-eventhub-py/scenarios.yaml +++ b/tests/scenarios/azure-eventhub-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-eventhub-py/vally/eval.yaml b/tests/scenarios/azure-eventhub-py/vally/eval.yaml new file mode 100644 index 00000000..8c48831e --- /dev/null +++ b/tests/scenarios/azure-eventhub-py/vally/eval.yaml @@ -0,0 +1,140 @@ +name: azure-eventhub-py-eval +description: Vally evaluation suite for azure-eventhub-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: producer_batch_send + prompt: |- + Create a synchronous Event Hub producer that authenticates with + DefaultAzureCredential, creates a batch, handles batch size limits, + and sends the batch. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: consumer_receive_with_checkpoint + prompt: |- + Create an Event Hub consumer that uses DefaultAzureCredential and + receives events from the beginning of the stream while checkpointing + each event. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-eventhub-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventhub-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..8c48831e --- /dev/null +++ b/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-eventhub-py-eval +description: Vally evaluation suite for azure-eventhub-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: producer_batch_send + prompt: |- + Create a synchronous Event Hub producer that authenticates with + DefaultAzureCredential, creates a batch, handles batch size limits, + and sends the batch. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: consumer_receive_with_checkpoint + prompt: |- + Create an Event Hub consumer that uses DefaultAzureCredential and + receives events from the beginning of the stream while checkpointing + each event. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-eventhub-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventhub-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..cb6f64b4 --- /dev/null +++ b/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-eventhub-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventhub-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml index ef73d6e5..5a43e34b 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-eventhub-producer-consumer @@ -54,7 +54,7 @@ stimuli: - "Builder pattern used for both client initialization." - "StartLocation configuration for consumer start position (Earliest/Latest)." - "Proper extraction of event body data from stream." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml index e1999661..372ce2bd 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-eventhub-producer-consumer @@ -54,7 +54,7 @@ stimuli: - "Builder pattern used for both client initialization." - "StartLocation configuration for consumer start position (Earliest/Latest)." - "Proper extraction of event body data from stream." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -78,7 +78,7 @@ stimuli: - type: prompt name: "EventHub producer-consumer completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - ProducerClient creation with builder pattern - Single event sending: producer.send_event(vec![...], None).await @@ -159,7 +159,7 @@ stimuli: - name: ProducerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ProducerClient created via ProducerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -168,7 +168,7 @@ stimuli: - name: ConsumerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ConsumerClient created via ConsumerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -177,7 +177,7 @@ stimuli: - name: Event Stream Processing type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Event stream obtained via receiver.stream_events() Stream processing with while let Some(event_result) = stream.next().await @@ -186,7 +186,7 @@ stimuli: - name: Credential Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses DeveloperToolsCredential for local dev or ManagedIdentityCredential for production Credential is cloned when passed to multiple clients diff --git a/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml index 6ea37bd7..68d5501a 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml @@ -7,9 +7,8 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: + timeout: 60m + model: claude-sonnet-4.6 grader_model: gpt-5.5stimuli: - name: rust-eventhub-producer-consumer prompt: | Write a Rust program named rust-eventhub-demo that demonstrates producer and consumer patterns @@ -54,7 +53,7 @@ stimuli: - "Builder pattern used for both client initialization." - "StartLocation configuration for consumer start position (Earliest/Latest)." - "Proper extraction of event body data from stream." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -84,7 +83,7 @@ stimuli: - type: prompt name: "EventHub producer-consumer completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - ProducerClient creation with builder pattern - Single event sending: producer.send_event(vec![...], None).await @@ -165,7 +164,7 @@ stimuli: - name: ProducerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ProducerClient created via ProducerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -174,7 +173,7 @@ stimuli: - name: ConsumerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ConsumerClient created via ConsumerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -183,7 +182,7 @@ stimuli: - name: Event Stream Processing type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Event stream obtained via receiver.stream_events() Stream processing with while let Some(event_result) = stream.next().await @@ -192,7 +191,7 @@ stimuli: - name: Credential Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses DeveloperToolsCredential for local dev or ManagedIdentityCredential for production Credential is cloned when passed to multiple clients diff --git a/tests/scenarios/azure-identity-py/scenarios.yaml b/tests/scenarios/azure-identity-py/scenarios.yaml index 0e1289b5..13571309 100644 --- a/tests/scenarios/azure-identity-py/scenarios.yaml +++ b/tests/scenarios/azure-identity-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-identity-py/vally/eval.yaml b/tests/scenarios/azure-identity-py/vally/eval.yaml new file mode 100644 index 00000000..d8054b08 --- /dev/null +++ b/tests/scenarios/azure-identity-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-identity-py-eval +description: Vally evaluation suite for azure-identity-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: default_credential_basic + prompt: |- + Show a minimal example using DefaultAzureCredential to request a token + for https://management.azure.com/.default. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: managed_identity_user_assigned + prompt: |- + Create a ManagedIdentityCredential using a user-assigned client ID from + AZURE_CLIENT_ID environment variable. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-identity-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-identity-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-identity-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-identity-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d8054b08 --- /dev/null +++ b/tests/scenarios/azure-identity-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-identity-py-eval +description: Vally evaluation suite for azure-identity-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: default_credential_basic + prompt: |- + Show a minimal example using DefaultAzureCredential to request a token + for https://management.azure.com/.default. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: managed_identity_user_assigned + prompt: |- + Create a ManagedIdentityCredential using a user-assigned client ID from + AZURE_CLIENT_ID environment variable. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-identity-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-identity-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-identity-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-identity-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..0c55d9b6 --- /dev/null +++ b/tests/scenarios/azure-identity-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-identity-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-identity-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml index 7f12fdc5..66321378 100644 --- a/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml +++ b/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-identity-credentials @@ -63,7 +63,7 @@ stimuli: - "Async/await with Tokio runtime." - "Code compiles without warnings." - "Result-based error propagation with ? operator." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-identity-rust/vally/eval.yaml b/tests/scenarios/azure-identity-rust/vally/eval.yaml index eacdf4f2..676ac4f3 100644 --- a/tests/scenarios/azure-identity-rust/vally/eval.yaml +++ b/tests/scenarios/azure-identity-rust/vally/eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-identity-credentials @@ -63,7 +63,7 @@ stimuli: - "Async/await with Tokio runtime." - "Code compiles without warnings." - "Result-based error propagation with ? operator." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -85,7 +85,7 @@ stimuli: - type: prompt name: "Identity credential completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - DeveloperToolsCredential creation and usage - ManagedIdentityCredential creation and usage @@ -164,7 +164,7 @@ stimuli: - name: DeveloperToolsCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > DeveloperToolsCredential created via DeveloperToolsCredential::new(None)? Attempts Azure CLI (az login) first, then Azure Developer CLI (azd auth login) @@ -172,7 +172,7 @@ stimuli: - name: ManagedIdentityCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ManagedIdentityCredential created via ManagedIdentityCredential::new(None)? System-assigned managed identity for Azure-hosted workloads @@ -181,7 +181,7 @@ stimuli: - name: ClientSecretCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ClientSecretCredential created with three parameters: tenant_id (string), client_id (string), client_secret (string) @@ -190,7 +190,7 @@ stimuli: - name: Credential Cloning type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Credentials are Arc-wrapped and support cloning Pass credential.clone() to multiple client constructors @@ -199,7 +199,7 @@ stimuli: - name: TokenCredential Trait type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > All credential types implement TokenCredential trait Allows generic code that accepts any credential type @@ -207,7 +207,7 @@ stimuli: - name: Environment Variable Configuration type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > AZURE_TENANT_ID: Tenant ID for service principal auth AZURE_CLIENT_ID: Client/application ID @@ -216,7 +216,7 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Credential creation returns Result Missing or invalid credentials generate errors @@ -225,7 +225,7 @@ stimuli: - name: Scenario Documentation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Code includes comments explaining when each credential is appropriate DeveloperToolsCredential for local development diff --git a/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml index 07b4383d..32d77321 100644 --- a/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml @@ -7,9 +7,8 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: + timeout: 60m + model: claude-sonnet-4.6 grader_model: gpt-5.5stimuli: - name: rust-identity-credentials prompt: | Write a Rust program named rust-identity-demo that demonstrates all major credential @@ -63,7 +62,7 @@ stimuli: - "Async/await with Tokio runtime." - "Code compiles without warnings." - "Result-based error propagation with ? operator." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -91,7 +90,7 @@ stimuli: - type: prompt name: "Identity credential completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - DeveloperToolsCredential creation and usage - ManagedIdentityCredential creation and usage @@ -148,7 +147,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 3m - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -159,19 +158,19 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 3m - type: run-command name: "Cargo Build" config: command: cargo args: - "build" - timeout: 120s + timeout: 3m - name: DeveloperToolsCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > DeveloperToolsCredential created via DeveloperToolsCredential::new(None)? Attempts Azure CLI (az login) first, then Azure Developer CLI (azd auth login) @@ -179,7 +178,7 @@ stimuli: - name: ManagedIdentityCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ManagedIdentityCredential created via ManagedIdentityCredential::new(None)? System-assigned managed identity for Azure-hosted workloads @@ -188,7 +187,7 @@ stimuli: - name: ClientSecretCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ClientSecretCredential created with three parameters: tenant_id (string), client_id (string), client_secret (string) @@ -197,7 +196,7 @@ stimuli: - name: Credential Cloning type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Credentials are Arc-wrapped and support cloning Pass credential.clone() to multiple client constructors @@ -206,7 +205,7 @@ stimuli: - name: TokenCredential Trait type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > All credential types implement TokenCredential trait Allows generic code that accepts any credential type @@ -214,7 +213,7 @@ stimuli: - name: Environment Variable Configuration type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > AZURE_TENANT_ID: Tenant ID for service principal auth AZURE_CLIENT_ID: Client/application ID @@ -223,7 +222,7 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Credential creation returns Result Missing or invalid credentials generate errors @@ -232,7 +231,7 @@ stimuli: - name: Scenario Documentation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Code includes comments explaining when each credential is appropriate DeveloperToolsCredential for local development diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml index 811e9217..52d63e60 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-keyvault-certificates-operations @@ -71,7 +71,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml index 37a4e92b..de8636a0 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-keyvault-certificates-operations @@ -71,7 +71,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -93,7 +93,7 @@ stimuli: - type: prompt name: "Key Vault certificates completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - CertificateClient creation with vault URL and credential - CertificatePolicy with X509CertificateProperties @@ -177,7 +177,7 @@ stimuli: - name: CertificateClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CertificateClient created via CertificateClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -185,7 +185,7 @@ stimuli: - name: Certificate Policy Definition type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CertificatePolicy includes x509_certificate_properties and issuer_parameters X509CertificateProperties has subject: Some("CN=example.com".into()) @@ -193,7 +193,7 @@ stimuli: - name: Long-Running Operation Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > begin_create_certificate() returns Poller Poller implements IntoFuture trait @@ -202,7 +202,7 @@ stimuli: - name: Certificate Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Get: client.get_certificate(name, None).await?.into_model()? Update: client.update_certificate_properties(name, params.try_into()?, None).await? @@ -211,7 +211,7 @@ stimuli: - name: Certificate Properties Update type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > UpdateCertificatePropertiesParameters with tags: Some(HashMap) Tags stored as Vec of key-value pairs @@ -220,7 +220,7 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(cert) = pager.try_next().await? ResourceExt provides cert.resource_id()? with name and version @@ -228,7 +228,7 @@ stimuli: - name: Certificate Key Integration type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Certificates contain embedded keys for signing operations KeyClient can sign with certificate name as key reference @@ -236,7 +236,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O awaited including LRO completion diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml index 9ce9fcef..fc22fd10 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml @@ -7,9 +7,8 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: + timeout: 60m + model: claude-sonnet-4.6 grader_model: gpt-5.5stimuli: - name: rust-keyvault-certificates-operations prompt: | Write a Rust program named rust-keyvault-certs that demonstrates X.509 certificate @@ -71,7 +70,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -99,7 +98,7 @@ stimuli: - type: prompt name: "Key Vault certificates completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - CertificateClient creation with vault URL and credential - CertificatePolicy with X509CertificateProperties @@ -126,6 +125,7 @@ stimuli: - "+nightly" - "-Zscript" - ".vally/tools/check-azure-crates.rs" + timeout: 3m - name: TokenCredential Authentication type: program weight: 1.0 @@ -160,7 +160,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 3m - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -171,7 +171,7 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 3m - type: run-command name: "Cargo Build" config: @@ -183,7 +183,7 @@ stimuli: - name: CertificateClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CertificateClient created via CertificateClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -191,7 +191,7 @@ stimuli: - name: Certificate Policy Definition type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CertificatePolicy includes x509_certificate_properties and issuer_parameters X509CertificateProperties has subject: Some("CN=example.com".into()) @@ -199,7 +199,7 @@ stimuli: - name: Long-Running Operation Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > begin_create_certificate() returns Poller Poller implements IntoFuture trait @@ -208,7 +208,7 @@ stimuli: - name: Certificate Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Get: client.get_certificate(name, None).await?.into_model()? Update: client.update_certificate_properties(name, params.try_into()?, None).await? @@ -217,7 +217,7 @@ stimuli: - name: Certificate Properties Update type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > UpdateCertificatePropertiesParameters with tags: Some(HashMap) Tags stored as Vec of key-value pairs @@ -226,7 +226,7 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(cert) = pager.try_next().await? ResourceExt provides cert.resource_id()? with name and version @@ -234,7 +234,7 @@ stimuli: - name: Certificate Key Integration type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Certificates contain embedded keys for signing operations KeyClient can sign with certificate name as key reference @@ -242,7 +242,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O awaited including LRO completion diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml index 11c078df..9362945e 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-keyvault-keys-operations @@ -70,7 +70,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml index b927c364..bdda5248 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-keyvault-keys-operations @@ -70,7 +70,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -92,7 +92,7 @@ stimuli: - type: prompt name: "Key Vault keys completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - KeyClient creation with vault URL and credential - Create EC keys with CurveName::P256 @@ -175,7 +175,7 @@ stimuli: - name: KeyClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > KeyClient created via KeyClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -183,7 +183,7 @@ stimuli: - name: Key Creation Patterns type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > EC keys: CreateKeyParameters with kty: Some(KeyType::Ec), curve: Some(CurveName::P256) RSA keys: CreateKeyParameters with kty: Some(KeyType::Rsa), key_size: Some(2048) @@ -191,7 +191,7 @@ stimuli: - name: Key Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Get: client.get_key(name, None).await?.into_model()? Update: client.update_key_properties(name, params.try_into()?, None).await? @@ -200,7 +200,7 @@ stimuli: - name: Envelope Encryption Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Wrap: client.wrap_key(key_name, params.try_into()?, None).await? Parameters include algorithm: Some(EncryptionAlgorithm::RsaOaep256) and value: Some(dek) @@ -210,7 +210,7 @@ stimuli: - name: Key Version Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Extract key version from response via resource_id()? Version stored from wrap operation for later unwrap @@ -219,7 +219,7 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(key) = pager.try_next().await? ResourceExt provides key.resource_id()? with name and version @@ -227,7 +227,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O awaited (client operations, wrapping, unwrapping) diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml index 3cdaaa30..fcb452be 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml @@ -7,9 +7,8 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: + timeout: 60m + model: claude-sonnet-4.6 grader_model: gpt-5.5stimuli: - name: rust-keyvault-keys-operations prompt: | Write a Rust program named rust-keyvault-keys that demonstrates cryptographic key @@ -70,7 +69,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -98,7 +97,7 @@ stimuli: - type: prompt name: "Key Vault keys completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - KeyClient creation with vault URL and credential - Create EC keys with CurveName::P256 @@ -181,7 +180,7 @@ stimuli: - name: KeyClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > KeyClient created via KeyClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -189,7 +188,7 @@ stimuli: - name: Key Creation Patterns type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > EC keys: CreateKeyParameters with kty: Some(KeyType::Ec), curve: Some(CurveName::P256) RSA keys: CreateKeyParameters with kty: Some(KeyType::Rsa), key_size: Some(2048) @@ -197,7 +196,7 @@ stimuli: - name: Key Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Get: client.get_key(name, None).await?.into_model()? Update: client.update_key_properties(name, params.try_into()?, None).await? @@ -206,7 +205,7 @@ stimuli: - name: Envelope Encryption Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Wrap: client.wrap_key(key_name, params.try_into()?, None).await? Parameters include algorithm: Some(EncryptionAlgorithm::RsaOaep256) and value: Some(dek) @@ -216,7 +215,7 @@ stimuli: - name: Key Version Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Extract key version from response via resource_id()? Version stored from wrap operation for later unwrap @@ -225,7 +224,7 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(key) = pager.try_next().await? ResourceExt provides key.resource_id()? with name and version @@ -233,7 +232,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O awaited (client operations, wrapping, unwrapping) diff --git a/tests/scenarios/azure-keyvault-py/scenarios.yaml b/tests/scenarios/azure-keyvault-py/scenarios.yaml index 90d59178..89b9f1b5 100644 --- a/tests/scenarios/azure-keyvault-py/scenarios.yaml +++ b/tests/scenarios/azure-keyvault-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-keyvault-py/vally/eval.yaml b/tests/scenarios/azure-keyvault-py/vally/eval.yaml new file mode 100644 index 00000000..a34cd396 --- /dev/null +++ b/tests/scenarios/azure-keyvault-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-keyvault-py-eval +description: Vally evaluation suite for azure-keyvault-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: secret_crud_basic + prompt: |- + Create a basic Azure Key Vault secrets example that authenticates with + DefaultAzureCredential, sets a secret, gets it, then deletes and purges it. + Use AZURE_KEYVAULT_URL for the vault URL. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and secret/key/certificate operations in try/except blocks + * Handle: ResourceNotFoundError, HttpResponseError, ClientAuthenticationError, AzureError + * Example: try: secret = client.get_secret(name) except ResourceNotFoundError: print('Secret not found') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: list_secret_versions + prompt: |- + List all secret names and all versions for a specific secret using + SecretClient.list_properties_of_secrets and list_properties_of_secret_versions. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and secret/key/certificate operations in try/except blocks + * Handle: ResourceNotFoundError, HttpResponseError, ClientAuthenticationError, AzureError + * Example: try: secret = client.get_secret(name) except ResourceNotFoundError: print('Secret not found') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-keyvault-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-keyvault-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..a369f4b9 --- /dev/null +++ b/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-keyvault-py-eval +description: Vally evaluation suite for azure-keyvault-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: secret_crud_basic + prompt: |- + Create a basic Azure Key Vault secrets example that authenticates with + DefaultAzureCredential, sets a secret, gets it, then deletes and purges it. + Use AZURE_KEYVAULT_URL for the vault URL. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and secret/key/certificate operations in try/except blocks + * Handle: ResourceNotFoundError, HttpResponseError, ClientAuthenticationError, AzureError + * Example: try: secret = client.get_secret(name) except ResourceNotFoundError: print('Secret not found') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: list_secret_versions + prompt: |- + List all secret names and all versions for a specific secret using + SecretClient.list_properties_of_secrets and list_properties_of_secret_versions. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and secret/key/certificate operations in try/except blocks + * Handle: ResourceNotFoundError, HttpResponseError, ClientAuthenticationError, AzureError + * Example: try: secret = client.get_secret(name) except ResourceNotFoundError: print('Secret not found') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-keyvault-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-keyvault-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..706ceefa --- /dev/null +++ b/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-keyvault-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-keyvault-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml index 7be891f3..b7c15da2 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-keyvault-secrets-operations @@ -63,7 +63,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -104,7 +104,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 180s - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -115,14 +115,14 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 300s - type: run-command name: "Cargo Build" config: command: cargo args: - "build" - timeout: 120s + timeout: 300s - name: SecretClient Creation type: prompt diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml index 632382dd..01d85978 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-keyvault-secrets-operations @@ -63,7 +63,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -85,7 +85,7 @@ stimuli: - type: prompt name: "Key Vault secrets completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - SecretClient creation with vault URL and credential - Set secret with SetSecretParameters including value @@ -142,7 +142,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 180s - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -153,19 +153,19 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 300s - type: run-command name: "Cargo Build" config: command: cargo args: - "build" - timeout: 120s + timeout: 300s - name: SecretClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > SecretClient created via SecretClient::new(vault_url, credential, None) Vault URL is vault_name.vault.azure.net in https protocol @@ -173,7 +173,7 @@ stimuli: - name: Secret CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Set: client.set_secret(name, params.try_into()?, None).await? Get: client.get_secret(name, None).await?.into_model()? @@ -182,7 +182,7 @@ stimuli: - name: SetSecretParameters Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > SetSecretParameters created with value: Some(value.into()) Content type can be set via content_type: Some("text/plain".into()) @@ -190,7 +190,7 @@ stimuli: - name: Listing with Pagination type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List secrets returns Pager via client.list_secret_properties(None)? Iterate with let mut pager = ...; while let Some(secret) = pager.try_next().await? @@ -199,7 +199,7 @@ stimuli: - name: Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Use ResourceExt trait via secret.resource_id()? Returns ResourceId with name and version fields @@ -207,7 +207,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O operations awaited diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml index d12448ac..01bce326 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml @@ -7,9 +7,8 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: + timeout: 60m + model: claude-sonnet-4.6 grader_model: gpt-5.5stimuli: - name: rust-keyvault-secrets-operations prompt: | Write a Rust program named rust-keyvault-secrets that demonstrates secure secret @@ -63,7 +62,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -91,7 +90,7 @@ stimuli: - type: prompt name: "Key Vault secrets completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - SecretClient creation with vault URL and credential - Set secret with SetSecretParameters including value @@ -149,7 +148,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 180s - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -160,19 +159,19 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 300s - type: run-command name: "Cargo Build" config: command: cargo args: - "build" - timeout: 120s + timeout: 300s - name: SecretClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > SecretClient created via SecretClient::new(vault_url, credential, None) Vault URL is vault_name.vault.azure.net in https protocol @@ -180,7 +179,7 @@ stimuli: - name: Secret CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Set: client.set_secret(name, params.try_into()?, None).await? Get: client.get_secret(name, None).await?.into_model()? @@ -189,7 +188,7 @@ stimuli: - name: SetSecretParameters Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > SetSecretParameters created with value: Some(value.into()) Content type can be set via content_type: Some("text/plain".into()) @@ -197,7 +196,7 @@ stimuli: - name: Listing with Pagination type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List secrets returns Pager via client.list_secret_properties(None)? Iterate with let mut pager = ...; while let Some(secret) = pager.try_next().await? @@ -206,7 +205,7 @@ stimuli: - name: Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Use ResourceExt trait via secret.resource_id()? Returns ResourceId with name and version fields @@ -214,7 +213,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O operations awaited diff --git a/tests/scenarios/azure-messaging-webpubsubservice-py/scenarios.yaml b/tests/scenarios/azure-messaging-webpubsubservice-py/scenarios.yaml index 8a373689..853acd17 100644 --- a/tests/scenarios/azure-messaging-webpubsubservice-py/scenarios.yaml +++ b/tests/scenarios/azure-messaging-webpubsubservice-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-messaging-webpubsubservice-py/vally/eval.yaml b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/eval.yaml new file mode 100644 index 00000000..85fcd9c1 --- /dev/null +++ b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: azure-messaging-webpubsubservice-py-eval +description: Vally evaluation suite for azure-messaging-webpubsubservice-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: connection_string_broadcast + prompt: |- + Create a WebPubSubServiceClient using a connection string from environment + variables, then send a text and JSON message to all connections. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: default_credential_targeted_sends + prompt: |- + Use DefaultAzureCredential with endpoint and hub from environment variables, + then send a message to a user and to a group. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-messaging-webpubsubservice-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..b54aee1f --- /dev/null +++ b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-messaging-webpubsubservice-py-eval +description: Vally evaluation suite for azure-messaging-webpubsubservice-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: connection_string_broadcast + prompt: |- + Create a WebPubSubServiceClient using a connection string from environment + variables, then send a text and JSON message to all connections. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: default_credential_targeted_sends + prompt: |- + Use DefaultAzureCredential with endpoint and hub from environment variables, + then send a message to a user and to a group. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-messaging-webpubsubservice-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..dade853e --- /dev/null +++ b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-messaging-webpubsubservice-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-mgmt-apicenter-py/scenarios.yaml b/tests/scenarios/azure-mgmt-apicenter-py/scenarios.yaml index 2a542cbf..2ff287af 100644 --- a/tests/scenarios/azure-mgmt-apicenter-py/scenarios.yaml +++ b/tests/scenarios/azure-mgmt-apicenter-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 @@ -23,7 +23,7 @@ scenarios: forbidden_patterns: - "hardcoded" - "xxxxx" - - "subscription_id=\"" + - 'subscription_id="' - "credential = Cli" tags: - basic @@ -34,22 +34,22 @@ scenarios: from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient from azure.mgmt.apicenter.models import Service - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + service = client.services.create_or_update( resource_group_name=resource_group, service_name="my-api-center", resource=Service(location="eastus"), ) - + print(f"Created service: {service.name}") # API Registration Workflow @@ -62,7 +62,7 @@ scenarios: - "create_or_update" - "ApiKind.REST" - "LifecycleStage" - - "workspace_name=\"default\"" + - 'workspace_name="default"' - "Api(" - "kind=" - "state=" @@ -80,16 +80,16 @@ scenarios: from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient from azure.mgmt.apicenter.models import Api, ApiKind, LifecycleStage - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + api = client.apis.create_or_update( resource_group_name=resource_group, service_name="my-api-center", @@ -102,7 +102,7 @@ scenarios: summary="API for managing pet store", ), ) - + print(f"Registered API: {api.title}") # API Version and Definition @@ -116,11 +116,12 @@ scenarios: - "api_definitions.create_or_update" - "ApiVersion(" - "ApiDefinition(" - - "specification=" - - "VersionStage" + - "ApiVersionProperties(" + - "ApiDefinitionProperties(" + - "begin_import_specification" - "workspace_name=" forbidden_patterns: - - "import_specification" + # - "import_specification" # Should be forbidden because the actual API is `begin_import_specification` - "swagger_url=" - "workspace_name=None" - "version_name=None" @@ -130,54 +131,68 @@ scenarios: - governance mock_response: | import os + from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient from azure.mgmt.apicenter.models import ( - ApiVersion, + ApiSpecImportRequest, + ApiSpecImportSourceFormat, ApiDefinition, - VersionStage, - ExternalDocumentationProperties, + ApiDefinitionProperties, + ApiVersion, + ApiVersionProperties, + LifecycleStage, ) - - subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") - resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - - credential = DefaultAzureCredential() - client = ApiCenterMgmtClient( - subscription_id=subscription_id, + + credential = DefaultAzureCredential(require_envvar=True) + + with ApiCenterMgmtClient( credential=credential, - ) - - # Create API version - version = client.api_versions.create_or_update( - resource_group_name=resource_group, - service_name="my-api-center", - workspace_name="default", - api_name="pet-api", - version_name="v1.0", - resource=ApiVersion( - title="Version 1.0", - stage=VersionStage.PRODUCTION, - ), - ) - - # Create API definition with spec - definition = client.api_definitions.create_or_update( - resource_group_name=resource_group, - service_name="my-api-center", - workspace_name="default", - api_name="pet-api", - version_name="v1.0", - definition_name="openapi", - resource=ApiDefinition( - title="OpenAPI Definition", - specification=ExternalDocumentationProperties( - url="https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/v3.0/petstore.json" + subscription_id=os.environ["AZURE_SUBSCRIPTION_ID"], + ) as client: + version = client.api_versions.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1.0", + resource=ApiVersion( + properties=ApiVersionProperties( + title="Version 1.0", + lifecycle_stage=LifecycleStage.PRODUCTION, + ), ), - ), - ) - - print(f"Created version: {version.title}, Definition: {definition.title}") + ) + + client.api_definitions.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1.0", + definition_name="openapi", + resource=ApiDefinition( + properties=ApiDefinitionProperties( + title="OpenAPI Definition", + description="OpenAPI 3.0 specification", + ), + ), + ) + + client.api_definitions.begin_import_specification( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1.0", + definition_name="openapi", + body=ApiSpecImportRequest( + format=ApiSpecImportSourceFormat.INLINE, + value='{"openapi":"3.0.0","info":{"title":"My API","version":"1.0","description":"Production OpenAPI specification for My API"},"paths":{}}', + ), + ).result() + + print(f"Created API version and imported specification: {version.title}") # Environment and Deployment Tracking - name: environment_and_deployment @@ -194,7 +209,6 @@ scenarios: - "deployment_name=" - "workspace_name=" forbidden_patterns: - - "environment_id=" - "deployment_id=" - "workspace_name=None" - "environment_name=None" @@ -204,55 +218,64 @@ scenarios: - tracking - governance mock_response: | - import os + from os import environ + from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient - from azure.mgmt.apicenter.models import ( - Environment, - Deployment, - DeploymentState, - EnvironmentKind, - ) - - subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") - resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - - credential = DefaultAzureCredential() - client = ApiCenterMgmtClient( - subscription_id=subscription_id, + from azure.mgmt.apicenter.models import Deployment, DeploymentState, Environment, EnvironmentKind + + RESOURCE_GROUP_NAME = "my-resource-group" + SERVICE_NAME = "my-api-center" + WORKSPACE_NAME = "default" + API_NAME = "my-api" + VERSION_NAME = "v1" + DEFINITION_NAME = "openapi" + ENVIRONMENT_NAME = "production" + DEPLOYMENT_NAME = "prod-deployment" + + credential = DefaultAzureCredential(require_envvar=True) + + with ApiCenterMgmtClient( credential=credential, - ) - - # Create environment - environment = client.environments.create_or_update( - resource_group_name=resource_group, - service_name="my-api-center", - workspace_name="default", - environment_name="production", - resource=Environment( - kind=EnvironmentKind.PRODUCTION, - title="Production", - server={"url": "https://api.example.com"}, - ), - ) - - # Track deployment - deployment = client.deployments.create_or_update( - resource_group_name=resource_group, - service_name="my-api-center", - workspace_name="default", - deployment_name="prod-2024-01", - resource=Deployment( - environment_name="production", - api_name="pet-api", - api_version="v1.0", - definition_name="openapi", - state=DeploymentState.ACTIVE, - title="Production Deployment Jan 2024", - ), - ) - - print(f"Environment: {environment.title}, Deployment: {deployment.title}") + subscription_id=environ["AZURE_SUBSCRIPTION_ID"], + ) as client: + environment = client.environments.create_or_update( + resource_group_name=RESOURCE_GROUP_NAME, + service_name=SERVICE_NAME, + workspace_name=WORKSPACE_NAME, + environment_name=ENVIRONMENT_NAME, + resource=Environment( + title="Production", + description="Production environment", + kind=EnvironmentKind.PRODUCTION, + server={ + "type": "Azure API Management", + "management_portal_uri": ["https://portal.azure.com"], + }, + ), + ) + + deployment = client.deployments.create_or_update( + resource_group_name=RESOURCE_GROUP_NAME, + service_name=SERVICE_NAME, + workspace_name=WORKSPACE_NAME, + api_name=API_NAME, + deployment_name=DEPLOYMENT_NAME, + resource=Deployment( + title="Production Deployment", + description="Deployed to production API Management", + environment_id=f"/workspaces/{WORKSPACE_NAME}/environments/{ENVIRONMENT_NAME}", + definition_id=( + f"/workspaces/{WORKSPACE_NAME}/apis/{API_NAME}" + f"/versions/{VERSION_NAME}/definitions/{DEFINITION_NAME}" + ), + state=DeploymentState.ACTIVE, + server={"runtime_uri": ["https://api.example.com"]}, + ), + ) + + print(f"Environment tracked: {environment.name} ({environment.kind})") + print(f"Deployment tracked: {deployment.name} ({deployment.state})") # Metadata Governance - Define Custom Schema - name: metadata_governance_schema @@ -284,16 +307,16 @@ scenarios: MetadataSchema, MetadataSchemaProperty, ) - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + # Define custom metadata schema schema = client.metadata_schemas.create_or_update( resource_group_name=resource_group, @@ -317,7 +340,7 @@ scenarios: ] ), ) - + print(f"Created metadata schema: {schema.name}") # List and Search APIs with Filtering @@ -344,23 +367,23 @@ scenarios: import os from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + # List all APIs in workspace apis = client.apis.list( resource_group_name=resource_group, service_name="my-api-center", workspace_name="default", ) - + print("APIs in catalog:") for api in apis: print(f" - {api.name}: {api.title}") @@ -393,16 +416,16 @@ scenarios: from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient from azure.mgmt.apicenter.models import Api, LifecycleStage - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + # Get existing API api = client.apis.get( resource_group_name=resource_group, @@ -410,7 +433,7 @@ scenarios: workspace_name="default", api_name="old-api", ) - + # Update lifecycle stage to deprecated updated_api = client.apis.create_or_update( resource_group_name=resource_group, @@ -424,7 +447,7 @@ scenarios: summary="This API is deprecated. Use new-api instead.", ), ) - + print(f"Updated {api.title} to {updated_api.state}") # Complete Governance Workflow @@ -451,7 +474,7 @@ scenarios: - "hardcoded" - "xxxxx" - "workspace_name=None" - - "subscription_id=\"" + - 'subscription_id="' tags: - workflow - governance @@ -472,16 +495,16 @@ scenarios: VersionStage, ExternalDocumentationProperties, ) - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + # 1. Create API Center service service = client.services.create_or_update( resource_group_name=resource_group, @@ -489,7 +512,7 @@ scenarios: resource=Service(location="eastus"), ) print(f"1. Created service: {service.name}") - + # 2. Register API api = client.apis.create_or_update( resource_group_name=resource_group, @@ -504,7 +527,7 @@ scenarios: ), ) print(f"2. Registered API: {api.title}") - + # 3. Create version and spec version = client.api_versions.create_or_update( resource_group_name=resource_group, @@ -518,7 +541,7 @@ scenarios: ), ) print(f"3. Created version: {version.title}") - + # 4. Define custom governance schema schema = client.metadata_schemas.create_or_update( resource_group_name=resource_group, @@ -541,7 +564,7 @@ scenarios: ), ) print(f"4. Created governance schema: {schema.name}") - + # 5. List all APIs print("5. Registered APIs:") apis = client.apis.list( diff --git a/tests/scenarios/azure-mgmt-apicenter-py/vally/eval.yaml b/tests/scenarios/azure-mgmt-apicenter-py/vally/eval.yaml new file mode 100644 index 00000000..4c5e351a --- /dev/null +++ b/tests/scenarios/azure-mgmt-apicenter-py/vally/eval.yaml @@ -0,0 +1,140 @@ +name: azure-mgmt-apicenter-py-eval +description: Vally evaluation suite for azure-mgmt-apicenter-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_api_center_creation + prompt: |- + Create a basic Azure API Center service with proper authentication, + specify a location (eastus), and include resource cleanup. + Use environment variables for configuration. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: api_registration_workflow + prompt: |- + Register a REST API in the API Center with metadata. + Include the API kind (REST), lifecycle stage, and description. + Use the "default" workspace for API operations. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-apicenter-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..4c5e351a --- /dev/null +++ b/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-mgmt-apicenter-py-eval +description: Vally evaluation suite for azure-mgmt-apicenter-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_api_center_creation + prompt: |- + Create a basic Azure API Center service with proper authentication, + specify a location (eastus), and include resource cleanup. + Use environment variables for configuration. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: api_registration_workflow + prompt: |- + Register a REST API in the API Center with metadata. + Include the API kind (REST), lifecycle stage, and description. + Use the "default" workspace for API operations. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-apicenter-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..03f76966 --- /dev/null +++ b/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-mgmt-apicenter-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-mgmt-apimanagement-py/scenarios.yaml b/tests/scenarios/azure-mgmt-apimanagement-py/scenarios.yaml index c3c106dc..dc976ccd 100644 --- a/tests/scenarios/azure-mgmt-apimanagement-py/scenarios.yaml +++ b/tests/scenarios/azure-mgmt-apimanagement-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-mgmt-apimanagement-py/vally/eval.yaml b/tests/scenarios/azure-mgmt-apimanagement-py/vally/eval.yaml new file mode 100644 index 00000000..fba45a44 --- /dev/null +++ b/tests/scenarios/azure-mgmt-apimanagement-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-mgmt-apimanagement-py-eval +description: Vally evaluation suite for azure-mgmt-apimanagement-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_apim_client_creation + prompt: |- + Create a basic Azure API Management client with proper authentication + and environment variable handling. Show how to connect to an APIM service. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: apim_service_creation_with_sku + prompt: |- + Create an Azure API Management service with Developer SKU. + Include publisher email and name. Handle the long-running operation properly. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-apimanagement-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..fba45a44 --- /dev/null +++ b/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-mgmt-apimanagement-py-eval +description: Vally evaluation suite for azure-mgmt-apimanagement-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_apim_client_creation + prompt: |- + Create a basic Azure API Management client with proper authentication + and environment variable handling. Show how to connect to an APIM service. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: apim_service_creation_with_sku + prompt: |- + Create an Azure API Management service with Developer SKU. + Include publisher email and name. Handle the long-running operation properly. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-apimanagement-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..0dfe0d21 --- /dev/null +++ b/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-mgmt-apimanagement-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-mgmt-botservice-py/scenarios.yaml b/tests/scenarios/azure-mgmt-botservice-py/scenarios.yaml index 70034961..12f44a57 100644 --- a/tests/scenarios/azure-mgmt-botservice-py/scenarios.yaml +++ b/tests/scenarios/azure-mgmt-botservice-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-mgmt-botservice-py/vally/eval.yaml b/tests/scenarios/azure-mgmt-botservice-py/vally/eval.yaml new file mode 100644 index 00000000..9a8ce29a --- /dev/null +++ b/tests/scenarios/azure-mgmt-botservice-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-mgmt-botservice-py-eval +description: Vally evaluation suite for azure-mgmt-botservice-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_bot_creation + prompt: |- + Create a basic Azure Bot Service using the management SDK. + Include proper authentication, bot creation with F0 SKU, and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: bot_retrieval_operations + prompt: |- + Write code to retrieve bot information from Azure Bot Service. + Include getting a specific bot, listing bots in a resource group, + and accessing bot properties. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-botservice-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..9a8ce29a --- /dev/null +++ b/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-mgmt-botservice-py-eval +description: Vally evaluation suite for azure-mgmt-botservice-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_bot_creation + prompt: |- + Create a basic Azure Bot Service using the management SDK. + Include proper authentication, bot creation with F0 SKU, and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: bot_retrieval_operations + prompt: |- + Write code to retrieve bot information from Azure Bot Service. + Include getting a specific bot, listing bots in a resource group, + and accessing bot properties. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-botservice-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..6deaacba --- /dev/null +++ b/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-mgmt-botservice-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-mgmt-fabric-py/scenarios.yaml b/tests/scenarios/azure-mgmt-fabric-py/scenarios.yaml index 97f10fbc..24737b6b 100644 --- a/tests/scenarios/azure-mgmt-fabric-py/scenarios.yaml +++ b/tests/scenarios/azure-mgmt-fabric-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-mgmt-fabric-py/vally/eval.yaml b/tests/scenarios/azure-mgmt-fabric-py/vally/eval.yaml new file mode 100644 index 00000000..218f4f22 --- /dev/null +++ b/tests/scenarios/azure-mgmt-fabric-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-mgmt-fabric-py-eval +description: Vally evaluation suite for azure-mgmt-fabric-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_creation + prompt: |- + Create a basic Azure Fabric Management client that connects to Azure + using DefaultAzureCredential. Include proper authentication and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: create_fabric_capacity + prompt: |- + Create a new Fabric capacity with the Azure Fabric Management SDK. + Include the SKU configuration, location, and proper LRO handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-fabric-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..218f4f22 --- /dev/null +++ b/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-mgmt-fabric-py-eval +description: Vally evaluation suite for azure-mgmt-fabric-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_creation + prompt: |- + Create a basic Azure Fabric Management client that connects to Azure + using DefaultAzureCredential. Include proper authentication and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: create_fabric_capacity + prompt: |- + Create a new Fabric capacity with the Azure Fabric Management SDK. + Include the SKU configuration, location, and proper LRO handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-fabric-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..caf6dfda --- /dev/null +++ b/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-mgmt-fabric-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-monitor-ingestion-py/scenarios.yaml b/tests/scenarios/azure-monitor-ingestion-py/scenarios.yaml index d6cef585..e7ff7431 100644 --- a/tests/scenarios/azure-monitor-ingestion-py/scenarios.yaml +++ b/tests/scenarios/azure-monitor-ingestion-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/eval.yaml b/tests/scenarios/azure-monitor-ingestion-py/vally/eval.yaml new file mode 100644 index 00000000..920cb145 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-monitor-ingestion-py-eval +description: Vally evaluation suite for azure-monitor-ingestion-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: sync_basic_upload + prompt: |- + Provide a minimal synchronous example that uploads custom logs. + Use DefaultAzureCredential, a DCE endpoint from AZURE_DCE_ENDPOINT, + and DCR rule/stream names from AZURE_DCR_RULE_ID and AZURE_DCR_STREAM_NAME. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: async_upload_with_context_manager + prompt: |- + Provide an async example using azure.monitor.ingestion.aio LogsIngestionClient. + Use async with and await upload. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-ingestion-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/full-output.txt b/tests/scenarios/azure-monitor-ingestion-py/vally/full-output.txt new file mode 100644 index 00000000..7de81cf1 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/full-output.txt @@ -0,0 +1,94 @@ +Running experiment 'azure-monitor-ingestion-py-skill-experiment' +Output: Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-35-17-490Z + +Loaded 4 stimuli from 2 eval(s) +Running with 5 workers + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ sync_basic_upload [variant: sonnet_baseline] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 29,139 + Turns 2 + Tool calls 2 + Wall time 27.4s + Errors 0 + Skills used 0 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The generated code is excellent across all rubric criteria. It uses the correct Azure SDK APIs with proper imports, reads all configuration from environment variables, provides complete executable code with a main() function and __main__ guard, handles all required exception types explicitly with no bare excepts, and follows modern Python conventions throughout. The conditional import pattern for ServiceError and ValidationError is a practical touch that handles SDK version differences. The code would be immediately runnable with the proper Azure environment configured. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ async_upload_with_context_manager [variant: sonnet_baseline] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 29,267 + Turns 2 + Tool calls 2 + Wall time 29.5s + Errors 0 + Skills used 0 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The generated code is excellent. It correctly uses the async Azure Monitor Logs Ingestion client with the exact patterns requested (async with, await upload), handles all four required exception types explicitly with no bare excepts, avoids hardcoded secrets by using DefaultAzureCredential, follows modern Python conventions throughout, and produces a complete, runnable file. The only minor point is that ENDPOINT and RULE_ID could be read from os.environ rather than being placeholder strings, but this is a common and acceptable pattern in Azure SDK examples. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ sync_basic_upload [variant: sonnet_skill] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 47,855 + Turns 3 + Tool calls 3 + Wall time 28.9s + Errors 0 + Skills used 1 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The solution is solid and largely idiomatic: it creates a runnable main.py, uses the correct SDK APIs and workflow, keeps all config in environment variables, uses a context manager for client lifecycle, and avoids bare excepts and wildcard imports. The main deductions are (1) the import of LogsUploadError from a private module (_exceptions), which is fragile across SDK versions, and (2) some explicitly requested exception types (ServiceError, ValidationError, AzureError) are not handled by nameΓÇöonly covered by the catch-all. These are minor but real gaps against the stated requirements. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ async_upload_with_context_manager [variant: sonnet_skill] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 48,624 + Turns 3 + Tool calls 3 + Wall time 33.2s + Errors 0 + Skills used 1 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The code is a solid, nearly complete async ingestion example that correctly demonstrates the Azure Monitor Ingestion async client with context managers, credential handling via environment variables, and a retry pattern for failed logs. It loses points mainly because `ValidationError` is entirely absent and `ServiceError` is not handled as required by the task's critical exception-handling specification. The private `_models` import is a minor style issue. Everything else ΓÇö structure, naming, control flow, and use of the correct async APIs ΓÇö is well-executed. + + All graders passed. + + Γ£ö azure-monitor-ingestion-py-eval [claude-sonnet-4.6] score: 91.7% (threshold: 75.0%) + + Γ£ö azure-monitor-ingestion-py-eval [claude-sonnet-4.6] score: 100.0% (threshold: 75.0%) + +Saved artifacts +Markdown ΓåÆ Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-35-17-490Z\report.md + +Γ£ö sonnet_skill: 1/1 evals passed +Γ£ö sonnet_baseline: 1/1 evals passed + +Run ID: 66a90c8e-0ddd-4cf9-a4b1-9b82fc1af30f +Output: Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-35-17-490Z diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..920cb145 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-monitor-ingestion-py-eval +description: Vally evaluation suite for azure-monitor-ingestion-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: sync_basic_upload + prompt: |- + Provide a minimal synchronous example that uploads custom logs. + Use DefaultAzureCredential, a DCE endpoint from AZURE_DCE_ENDPOINT, + and DCR rule/stream names from AZURE_DCR_RULE_ID and AZURE_DCR_STREAM_NAME. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: async_upload_with_context_manager + prompt: |- + Provide an async example using azure.monitor.ingestion.aio LogsIngestionClient. + Use async with and await upload. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-ingestion-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..bf835072 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-monitor-ingestion-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/vally-output.txt b/tests/scenarios/azure-monitor-ingestion-py/vally/vally-output.txt new file mode 100644 index 00000000..56c291f4 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/vally-output.txt @@ -0,0 +1,94 @@ +Running experiment 'azure-monitor-ingestion-py-skill-experiment' +Output: Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-40-22-287Z + +Loaded 4 stimuli from 2 eval(s) +Running with 5 workers + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ async_upload_with_context_manager [variant: sonnet_skill] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 47,643 + Turns 3 + Tool calls 3 + Wall time 27.4s + Errors 0 + Skills used 1 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The agent produced a well-structured, idiomatic async Python file that correctly uses the azure-monitor-ingestion async API, keeps secrets in environment variables, and provides explicit exception handling with no bare excepts. The only minor gap is that `ValidationError` is not explicitly handled and the outer except is a broad `Exception`, but these are minor issues that don't significantly detract from the quality of the solution. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ sync_basic_upload [variant: sonnet_baseline] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 28,687 + Turns 2 + Tool calls 2 + Wall time 23.2s + Errors 0 + Skills used 0 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The generated code is well-structured, complete, and runnable. It correctly uses the Azure Monitor Ingestion SDK with DefaultAzureCredential, reads all configuration from environment variables, and includes layered explicit exception handling covering all required types. The only notable weakness is importing from the private azure.monitor.ingestion._exceptions module rather than the public SDK surface, which is a minor but real best-practice concern. Apart from that one issue, the code is idiomatic Python and satisfies all the stated requirements. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ async_upload_with_context_manager [variant: sonnet_baseline] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 29,159 + Turns 2 + Tool calls 2 + Wall time 30.6s + Errors 0 + Skills used 0 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The output is a high-quality, idiomatic async Python example using the correct Azure SDK classes and patterns. It uses `async with`, `await upload`, proper context managers, meaningful exception handling with no bare `except`, and clean Python conventions. The two small shortcomings ΓÇö missing `ValidationError` handling and hardcoded endpoint/rule-id placeholders instead of environment variable reads ΓÇö prevent a perfect score but do not significantly diminish the practical utility or correctness of the example. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ sync_basic_upload [variant: sonnet_skill] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 47,828 + Turns 3 + Tool calls 3 + Wall time 29.3s + Errors 0 + Skills used 1 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The generated code is a high-quality, idiomatic Python example that correctly uses the Azure Monitor Ingestion SDK. It covers all required elements: env-var-based configuration, DefaultAzureCredential, the upload() API with an on_error callback, a context manager for client lifecycle, and explicit exception handling without bare excepts or wildcard imports. The only minor gap is that ValidationError and AzureError are not listed as named exception types (only caught by the broad fallback), which slightly misses the task's explicit requirement but does not compromise correctness or safety. + + All graders passed. + + Γ£ö azure-monitor-ingestion-py-eval [claude-sonnet-4.6] score: 100.0% (threshold: 75.0%) + + Γ£ö azure-monitor-ingestion-py-eval [claude-sonnet-4.6] score: 91.7% (threshold: 75.0%) + +Saved artifacts +Markdown ΓåÆ Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-40-22-287Z\report.md + +Γ£ö sonnet_skill: 1/1 evals passed +Γ£ö sonnet_baseline: 1/1 evals passed + +Run ID: 3fde020f-faf4-4f9c-9c34-e50919414d41 +Output: Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-40-22-287Z diff --git a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/scenarios.yaml b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/scenarios.yaml index a61c3d07..26dffd41 100644 --- a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/scenarios.yaml +++ b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/eval.yaml b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/eval.yaml new file mode 100644 index 00000000..693e4f50 --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-monitor-opentelemetry-exporter-py-eval +description: Vally evaluation suite for azure-monitor-opentelemetry-exporter-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: trace_exporter_basic + prompt: |- + Create a minimal trace exporter example using AzureMonitorTraceExporter. + Use BatchSpanProcessor, TracerProvider, and emit a span. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: metric_exporter_basic + prompt: |- + Show how to configure AzureMonitorMetricExporter with a + PeriodicExportingMetricReader and record a counter metric. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-opentelemetry-exporter-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..693e4f50 --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-monitor-opentelemetry-exporter-py-eval +description: Vally evaluation suite for azure-monitor-opentelemetry-exporter-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: trace_exporter_basic + prompt: |- + Create a minimal trace exporter example using AzureMonitorTraceExporter. + Use BatchSpanProcessor, TracerProvider, and emit a span. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: metric_exporter_basic + prompt: |- + Show how to configure AzureMonitorMetricExporter with a + PeriodicExportingMetricReader and record a counter metric. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-opentelemetry-exporter-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..9b591da9 --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-monitor-opentelemetry-exporter-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-monitor-opentelemetry-py/scenarios.yaml b/tests/scenarios/azure-monitor-opentelemetry-py/scenarios.yaml index 9e8abaf1..03c57c48 100644 --- a/tests/scenarios/azure-monitor-opentelemetry-py/scenarios.yaml +++ b/tests/scenarios/azure-monitor-opentelemetry-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-monitor-opentelemetry-py/vally/eval.yaml b/tests/scenarios/azure-monitor-opentelemetry-py/vally/eval.yaml new file mode 100644 index 00000000..c819bccb --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-monitor-opentelemetry-py-eval +description: Vally evaluation suite for azure-monitor-opentelemetry-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: one_line_setup + prompt: |- + Provide the minimal one-line setup for Azure Monitor OpenTelemetry in Python. + Use configure_azure_monitor and assume the connection string is in environment variables. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: explicit_connection_string + prompt: |- + Show configure_azure_monitor with an explicit connection string literal. + Include InstrumentationKey and IngestionEndpoint. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-opentelemetry-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..c819bccb --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-monitor-opentelemetry-py-eval +description: Vally evaluation suite for azure-monitor-opentelemetry-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: one_line_setup + prompt: |- + Provide the minimal one-line setup for Azure Monitor OpenTelemetry in Python. + Use configure_azure_monitor and assume the connection string is in environment variables. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: explicit_connection_string + prompt: |- + Show configure_azure_monitor with an explicit connection string literal. + Include InstrumentationKey and IngestionEndpoint. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-opentelemetry-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..dbf12bf2 --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-monitor-opentelemetry-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-monitor-query-py/scenarios.yaml b/tests/scenarios/azure-monitor-query-py/scenarios.yaml index 0ecca5dc..c0c43391 100644 --- a/tests/scenarios/azure-monitor-query-py/scenarios.yaml +++ b/tests/scenarios/azure-monitor-query-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-monitor-query-py/vally/eval.yaml b/tests/scenarios/azure-monitor-query-py/vally/eval.yaml new file mode 100644 index 00000000..d7da6e68 --- /dev/null +++ b/tests/scenarios/azure-monitor-query-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-monitor-query-py-eval +description: Vally evaluation suite for azure-monitor-query-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: logs_basic_query + prompt: |- + Provide a basic LogsQueryClient example that queries a Log Analytics workspace. + Use DefaultAzureCredential and a relative timespan. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: logs_timespan_tuple + prompt: |- + Show a LogsQueryClient example that uses a time range tuple with timezone-aware datetimes. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-query-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-query-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d7da6e68 --- /dev/null +++ b/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,137 @@ +name: azure-monitor-query-py-eval +description: Vally evaluation suite for azure-monitor-query-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: logs_basic_query + prompt: |- + Provide a basic LogsQueryClient example that queries a Log Analytics workspace. + Use DefaultAzureCredential and a relative timespan. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: logs_timespan_tuple + prompt: |- + Show a LogsQueryClient example that uses a time range tuple with timezone-aware datetimes. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-query-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-query-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..e2fefce1 --- /dev/null +++ b/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-monitor-query-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-query-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-search-documents-py/scenarios.yaml b/tests/scenarios/azure-search-documents-py/scenarios.yaml index 1e9bd997..7cd61531 100644 --- a/tests/scenarios/azure-search-documents-py/scenarios.yaml +++ b/tests/scenarios/azure-search-documents-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-search-documents-py/vally/eval.yaml b/tests/scenarios/azure-search-documents-py/vally/eval.yaml new file mode 100644 index 00000000..10154d41 --- /dev/null +++ b/tests/scenarios/azure-search-documents-py/vally/eval.yaml @@ -0,0 +1,140 @@ +name: azure-search-documents-py-eval +description: Vally evaluation suite for azure-search-documents-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_setup + prompt: |- + Create a basic Azure AI Search client using SearchClient with proper + authentication. Include connecting to an existing search index and + executing a simple search query. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: vector_search_basic + prompt: |- + Create a vector search example using Azure AI Search. + Generate embeddings for a query and search using VectorizedQuery. + Include proper vector configuration and search parameters. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-search-documents-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-search-documents-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..10154d41 --- /dev/null +++ b/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-search-documents-py-eval +description: Vally evaluation suite for azure-search-documents-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_setup + prompt: |- + Create a basic Azure AI Search client using SearchClient with proper + authentication. Include connecting to an existing search index and + executing a simple search query. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: vector_search_basic + prompt: |- + Create a vector search example using Azure AI Search. + Generate embeddings for a query and search using VectorizedQuery. + Include proper vector configuration and search parameters. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-search-documents-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-search-documents-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..6297c471 --- /dev/null +++ b/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-search-documents-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-search-documents-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-servicebus-py/scenarios.yaml b/tests/scenarios/azure-servicebus-py/scenarios.yaml index 69c7b216..6cdff8c6 100644 --- a/tests/scenarios/azure-servicebus-py/scenarios.yaml +++ b/tests/scenarios/azure-servicebus-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-servicebus-py/vally/eval.yaml b/tests/scenarios/azure-servicebus-py/vally/eval.yaml new file mode 100644 index 00000000..6aac7c92 --- /dev/null +++ b/tests/scenarios/azure-servicebus-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-servicebus-py-eval +description: Vally evaluation suite for azure-servicebus-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: sync_queue_send_receive + prompt: |- + Create a sync example that sends a message to a queue and receives it. + Use DefaultAzureCredential and context managers. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: async_queue_send_receive + prompt: |- + Create an async example that sends and receives messages from a queue. + Use async clients and await all operations. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-servicebus-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-servicebus-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..6aac7c92 --- /dev/null +++ b/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-servicebus-py-eval +description: Vally evaluation suite for azure-servicebus-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: sync_queue_send_receive + prompt: |- + Create a sync example that sends a message to a queue and receives it. + Use DefaultAzureCredential and context managers. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: async_queue_send_receive + prompt: |- + Create an async example that sends and receives messages from a queue. + Use async clients and await all operations. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-servicebus-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-servicebus-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..6c955f20 --- /dev/null +++ b/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-servicebus-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-servicebus-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml index 98656955..51880dbd 100644 --- a/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml +++ b/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-servicebus-messaging @@ -55,7 +55,7 @@ stimuli: - "Demonstrates both queue (point-to-point) and topic (pub-sub) patterns." - "Async/await with Tokio runtime and proper error handling." - "Code compiles without warnings and follows Rust idioms." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-servicebus-rust/vally/eval.yaml b/tests/scenarios/azure-servicebus-rust/vally/eval.yaml index 93e4e130..8a7ce938 100644 --- a/tests/scenarios/azure-servicebus-rust/vally/eval.yaml +++ b/tests/scenarios/azure-servicebus-rust/vally/eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-servicebus-messaging @@ -55,7 +55,7 @@ stimuli: - "Demonstrates both queue (point-to-point) and topic (pub-sub) patterns." - "Async/await with Tokio runtime and proper error handling." - "Code compiles without warnings and follows Rust idioms." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -77,7 +77,7 @@ stimuli: - type: prompt name: "Service Bus messaging completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - ServiceBusClient initialization with builder and open() - Queue sender/receiver creation @@ -156,7 +156,7 @@ stimuli: - name: ServiceBusClient Builder Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ServiceBusClient created via ServiceBusClient::builder() .open(namespace, credential).await? @@ -166,7 +166,7 @@ stimuli: - name: Message Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Messages created via Message::from(text) or Message::from(bytes) Send via sender.send_message(message, None).await? @@ -175,7 +175,7 @@ stimuli: - name: Message Settlement type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > After processing, call receiver.complete_message(&message, None).await? For retriable errors, use receiver.abandon_message(&message, None).await? @@ -184,7 +184,7 @@ stimuli: - name: Queue and Topic Patterns type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Queues demonstrated via create_sender/create_receiver on queue names Topics demonstrated via create_sender on topic names diff --git a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml index 6d37af5b..2d8ddb6b 100644 --- a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml @@ -7,9 +7,8 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: + timeout: 60m + model: claude-sonnet-4.6 grader_model: gpt-5.5stimuli: - name: rust-servicebus-messaging prompt: | Write a Rust program named rust-servicebus-ops that demonstrates Azure Service Bus @@ -55,7 +54,7 @@ stimuli: - "Demonstrates both queue (point-to-point) and topic (pub-sub) patterns." - "Async/await with Tokio runtime and proper error handling." - "Code compiles without warnings and follows Rust idioms." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -83,7 +82,7 @@ stimuli: - type: prompt name: "Service Bus messaging completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - ServiceBusClient initialization with builder and open() - Queue sender/receiver creation @@ -163,7 +162,7 @@ stimuli: - name: ServiceBusClient Builder Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ServiceBusClient created via ServiceBusClient::builder() .open(namespace, credential).await? @@ -173,7 +172,7 @@ stimuli: - name: Message Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Messages created via Message::from(text) or Message::from(bytes) Send via sender.send_message(message, None).await? @@ -182,7 +181,7 @@ stimuli: - name: Message Settlement type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > After processing, call receiver.complete_message(&message, None).await? For retriable errors, use receiver.abandon_message(&message, None).await? @@ -191,7 +190,7 @@ stimuli: - name: Queue and Topic Patterns type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Queues demonstrated via create_sender/create_receiver on queue names Topics demonstrated via create_sender on topic names diff --git a/tests/scenarios/azure-speech-to-text-rest-py/scenarios.yaml b/tests/scenarios/azure-speech-to-text-rest-py/scenarios.yaml index 397ff312..63f26827 100644 --- a/tests/scenarios/azure-speech-to-text-rest-py/scenarios.yaml +++ b/tests/scenarios/azure-speech-to-text-rest-py/scenarios.yaml @@ -1,6 +1,6 @@ config: max_tokens: 2000 - model: gpt-4 + model: gpt-5.5 temperature: 0.3 scenarios: - expected_patterns: diff --git a/tests/scenarios/azure-speech-to-text-rest-py/vally/eval.yaml b/tests/scenarios/azure-speech-to-text-rest-py/vally/eval.yaml new file mode 100644 index 00000000..b8a76cb4 --- /dev/null +++ b/tests/scenarios/azure-speech-to-text-rest-py/vally/eval.yaml @@ -0,0 +1,86 @@ +name: azure-speech-to-text-rest-py-eval +description: Vally evaluation suite for azure-speech-to-text-rest-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-speech-to-text-rest-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-speech-to-text-rest-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..b2484675 --- /dev/null +++ b/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,87 @@ +name: azure-speech-to-text-rest-py-eval +description: Vally evaluation suite for azure-speech-to-text-rest-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-speech-to-text-rest-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-speech-to-text-rest-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..bca2d438 --- /dev/null +++ b/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-speech-to-text-rest-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-blob-py/scenarios.yaml b/tests/scenarios/azure-storage-blob-py/scenarios.yaml index 0fdd1a07..36f505bd 100644 --- a/tests/scenarios/azure-storage-blob-py/scenarios.yaml +++ b/tests/scenarios/azure-storage-blob-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-storage-blob-py/vally/eval.yaml b/tests/scenarios/azure-storage-blob-py/vally/eval.yaml new file mode 100644 index 00000000..46e7f6e9 --- /dev/null +++ b/tests/scenarios/azure-storage-blob-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-storage-blob-py-eval +description: Vally evaluation suite for azure-storage-blob-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_blob_service_client + prompt: |- + Create a basic Azure Blob Storage example that authenticates with + DefaultAzureCredential and initializes a BlobServiceClient. Then + create a container client and create the container. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: upload_blob_with_blob_client + prompt: |- + Upload a local file to Azure Blob Storage using BlobClient. + Use overwrite=True when uploading. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-blob-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-blob-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..94ee8d0f --- /dev/null +++ b/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-storage-blob-py-eval +description: Vally evaluation suite for azure-storage-blob-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_blob_service_client + prompt: |- + Create a basic Azure Blob Storage example that authenticates with + DefaultAzureCredential and initializes a BlobServiceClient. Then + create a container client and create the container. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: upload_blob_with_blob_client + prompt: |- + Upload a local file to Azure Blob Storage using BlobClient. + Use overwrite=True when uploading. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-blob-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-blob-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..89f97928 --- /dev/null +++ b/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-storage-blob-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-blob-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml index b0248616..2c73994a 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-storage-keyvault-encryption-library @@ -39,7 +39,7 @@ stimuli: - "Uses async Rust with Tokio runtime and idiomatic Result-based error propagation." - "Code compiles and passes cargo check for the generated library project." - "Avoids hardcoded secrets in source code or committed configuration." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml index 0045654b..bd4bbfa7 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-crud-blobs-nobuild @@ -29,7 +29,7 @@ stimuli: - "Uses correct Azure client construction, response handling, and pager usage where applicable." - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml index 803bcd3f..dc972629 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-crud-blobs-nobuild @@ -29,7 +29,7 @@ stimuli: - "Uses correct Azure client construction, response handling, and pager usage where applicable." - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml index 7064a05d..1267e8a4 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-crud-blobs @@ -37,7 +37,7 @@ stimuli: - "Uses correct Azure client construction, response handling, and pager usage where applicable." - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -83,7 +83,7 @@ stimuli: - type: prompt name: "Prompt correctness and completeness of blob storage operations" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - The code should demonstrate an azure blob storage client with a storage account url and credential - Creating a container for container creation @@ -124,7 +124,7 @@ stimuli: - name: Idiomatic Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Error handling uses Result and the ? operator. Calls to .unwrap() are avoided or used only where failure @@ -137,7 +137,7 @@ stimuli: - name: Correct Client Construction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Service clients are constructed using the ::new() constructor with an endpoint string, credential, and Option parameter. @@ -146,7 +146,7 @@ stimuli: #[allow(clippy::needless_update)] annotation is used. - name: Pager and Stream Usage type: prompt - model: gpt-5.3-codex + model: claude-sonnet-4.6 weight: 1.0 prompt: > List operations return and consume azure_core::Pager using @@ -160,7 +160,7 @@ stimuli: - name: Response Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Calls to service clients correctly handle Response. When service methods return a model type, code calls into_model() to deserialize @@ -170,7 +170,7 @@ stimuli: - name: Long-Running Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Long-running operations use azure_core::Poller correctly. If a client calls into an API returning a Poller, code awaits @@ -181,7 +181,7 @@ stimuli: - name: Struct Initialization with Default type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > If a client initializes a structure for SDK types (client options, method options, model types) the code uses ..Default::default() to protect @@ -189,7 +189,7 @@ stimuli: - name: Ownership and Borrowing type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Code uses references (&str, &T) for borrowed data and avoids unnecessary .clone() calls. Credentials are shared via Arc or diff --git a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml index e04f29f1..1c5b6e4f 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml @@ -7,9 +7,8 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: + timeout: 60m + model: claude-sonnet-4.6 grader_model: gpt-5.5stimuli: - name: rust-crud-blobs prompt: | Write a Rust program named rust-blob that performs the following blob storage operations: @@ -37,7 +36,7 @@ stimuli: - "Uses correct Azure client construction, response handling, and pager usage where applicable." - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -61,8 +60,6 @@ stimuli: - "**/*.lib" - "**/*.wasm" environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-storage-blob-rust files: - src: ../../_shared/vally/tools/find-cargo-root.js dest: .vally/tools/find-cargo-root.js @@ -83,7 +80,7 @@ stimuli: - type: prompt name: "Prompt correctness and completeness of blob storage operations" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - The code should demonstrate an azure blob storage client with a storage account url and credential - Creating a container for container creation @@ -100,7 +97,7 @@ stimuli: command: cargo args: - "build" - timeout: 120s + timeout: 3m stderr_contains: "Finished" - type: run-command name: "Check Application" @@ -108,7 +105,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 3m - name: Rust Clippy Lints type: run-command weight: 0.5 @@ -150,7 +147,7 @@ stimuli: - name: Idiomatic Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Error handling uses Result and the ? operator. Calls to .unwrap() are avoided or used only where failure @@ -163,7 +160,7 @@ stimuli: - name: Correct Client Construction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Service clients are constructed using the ::new() constructor with an endpoint string, credential, and Option parameter. @@ -172,7 +169,7 @@ stimuli: #[allow(clippy::needless_update)] annotation is used. - name: Pager and Stream Usage type: prompt - model: gpt-5.3-codex + model: claude-sonnet-4.6 weight: 1.0 prompt: > List operations return and consume azure_core::Pager using @@ -186,7 +183,7 @@ stimuli: - name: Response Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Calls to service clients correctly handle Response. When service methods return a model type, code calls into_model() to deserialize @@ -196,7 +193,7 @@ stimuli: - name: Long-Running Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Long-running operations use azure_core::Poller correctly. If a client calls into an API returning a Poller, code awaits @@ -207,7 +204,7 @@ stimuli: - name: Struct Initialization with Default type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > If a client initializes a structure for SDK types (client options, method options, model types) the code uses ..Default::default() to protect @@ -215,7 +212,7 @@ stimuli: - name: Ownership and Borrowing type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Code uses references (&str, &T) for borrowed data and avoids unnecessary .clone() calls. Credentials are shared via Arc or @@ -231,46 +228,3 @@ stimuli: - "+nightly" - "-Zscript" - ".vally/tools/check-no-secrets.rs" - - name: Rust Cargo Trajectory Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - collect_trajectory_compiler_errors: true - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo build - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Check Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo check - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Clippy Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo clippy - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail diff --git a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml index 859e46e3..42bebb73 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml @@ -2,8 +2,6 @@ name: azure-storage-blob-rust-skill-experiment evals: - skill_effectiveness_eval.yaml - - eval-nobuild-experiment.yaml - - eval-keyvault-encryption.yaml vary: - /defaults/model diff --git a/tests/scenarios/azure-storage-file-datalake-py/scenarios.yaml b/tests/scenarios/azure-storage-file-datalake-py/scenarios.yaml index 4c86bd93..82b9eaba 100644 --- a/tests/scenarios/azure-storage-file-datalake-py/scenarios.yaml +++ b/tests/scenarios/azure-storage-file-datalake-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-storage-file-datalake-py/vally/eval.yaml b/tests/scenarios/azure-storage-file-datalake-py/vally/eval.yaml new file mode 100644 index 00000000..e79a127e --- /dev/null +++ b/tests/scenarios/azure-storage-file-datalake-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-storage-file-datalake-py-eval +description: Vally evaluation suite for azure-storage-file-datalake-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_service_client_and_filesystem + prompt: |- + Create a basic example that authenticates with DefaultAzureCredential, + creates a DataLakeServiceClient, and creates/gets a file system. + Include listing file systems. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: directory_operations + prompt: |- + Show how to create a directory, get a directory client, + create a subdirectory, and rename the directory. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-file-datalake-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..586dabfe --- /dev/null +++ b/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-storage-file-datalake-py-eval +description: Vally evaluation suite for azure-storage-file-datalake-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_service_client_and_filesystem + prompt: |- + Create a basic example that authenticates with DefaultAzureCredential, + creates a DataLakeServiceClient, and creates/gets a file system. + Include listing file systems. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: directory_operations + prompt: |- + Show how to create a directory, get a directory client, + create a subdirectory, and rename the directory. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-file-datalake-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..7ce7618d --- /dev/null +++ b/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-storage-file-datalake-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-file-share-py/scenarios.yaml b/tests/scenarios/azure-storage-file-share-py/scenarios.yaml index 1c518d79..f92a3111 100644 --- a/tests/scenarios/azure-storage-file-share-py/scenarios.yaml +++ b/tests/scenarios/azure-storage-file-share-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-storage-file-share-py/vally/eval.yaml b/tests/scenarios/azure-storage-file-share-py/vally/eval.yaml new file mode 100644 index 00000000..bb66ccbb --- /dev/null +++ b/tests/scenarios/azure-storage-file-share-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: azure-storage-file-share-py-eval +description: Vally evaluation suite for azure-storage-file-share-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: service_client_connection_string + prompt: |- + Create a ShareServiceClient using a connection string and demonstrate + creating a share, listing shares, and deleting the share. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: service_client_default_azure_credential + prompt: |- + Use DefaultAzureCredential with a ShareServiceClient and account_url. + Create a share client, create a directory, and clean up the share. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-file-share-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d8533d2f --- /dev/null +++ b/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-storage-file-share-py-eval +description: Vally evaluation suite for azure-storage-file-share-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: service_client_connection_string + prompt: |- + Create a ShareServiceClient using a connection string and demonstrate + creating a share, listing shares, and deleting the share. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: service_client_default_azure_credential + prompt: |- + Use DefaultAzureCredential with a ShareServiceClient and account_url. + Create a share client, create a directory, and clean up the share. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-file-share-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..d6d9022b --- /dev/null +++ b/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-storage-file-share-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-queue-py/scenarios.yaml b/tests/scenarios/azure-storage-queue-py/scenarios.yaml index 848489f5..fca23d17 100644 --- a/tests/scenarios/azure-storage-queue-py/scenarios.yaml +++ b/tests/scenarios/azure-storage-queue-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-storage-queue-py/vally/eval.yaml b/tests/scenarios/azure-storage-queue-py/vally/eval.yaml new file mode 100644 index 00000000..1b961036 --- /dev/null +++ b/tests/scenarios/azure-storage-queue-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: azure-storage-queue-py-eval +description: Vally evaluation suite for azure-storage-queue-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_queue_service_client + prompt: |- + Create a basic example using QueueServiceClient with DefaultAzureCredential. + Create a queue, list queues, and delete the queue. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: queue_client_send_receive_peek + prompt: |- + Create a QueueClient that sends a message, receives messages, + peeks messages, and deletes messages after processing. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-queue-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-queue-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d98d1a52 --- /dev/null +++ b/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-storage-queue-py-eval +description: Vally evaluation suite for azure-storage-queue-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_queue_service_client + prompt: |- + Create a basic example using QueueServiceClient with DefaultAzureCredential. + Create a queue, list queues, and delete the queue. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: queue_client_send_receive_peek + prompt: |- + Create a QueueClient that sends a message, receives messages, + peeks messages, and deletes messages after processing. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-queue-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-queue-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..d162882c --- /dev/null +++ b/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-storage-queue-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-queue-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml index a48e09f8..f6ed489e 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-queue-operations @@ -41,7 +41,7 @@ stimuli: - "Message delete operations use both message_id and pop_receipt from received messages." - "Peek operations demonstrate reading messages without consuming them from the queue." - "Avoids hardcoded credentials in source code." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml index 131b8112..2aa65334 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml @@ -7,7 +7,7 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-queue-operations @@ -41,7 +41,7 @@ stimuli: - "Message delete operations use both message_id and pop_receipt from received messages." - "Peek operations demonstrate reading messages without consuming them from the queue." - "Avoids hardcoded credentials in source code." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -77,7 +77,7 @@ stimuli: - type: prompt name: "Queue operations completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - QueueServiceClient initialization with endpoint URL and credential - QueueClient derived from service client @@ -154,7 +154,7 @@ stimuli: - name: QueueServiceClient Construction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > QueueServiceClient is created via QueueServiceClient::new() with three parameters: service_url (Url type), Some(credential), and None for options. @@ -162,7 +162,7 @@ stimuli: - name: Queue Message Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Send messages via queue_client.send_message(message.try_into()?, None).await?. Receive messages via queue_client.receive_messages(None).await? returning a response @@ -172,7 +172,7 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Queue operations are wrapped in Result-based error handling using ? operator. No .unwrap() calls except where failure is impossible. diff --git a/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml index f4ae4478..15268bdf 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml @@ -7,9 +7,8 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: + timeout: 60m + model: claude-sonnet-4.6 grader_model: gpt-5.5stimuli: - name: rust-queue-operations prompt: | Write a Rust program named rust-queue-ops that performs the following queue storage operations: @@ -41,7 +40,7 @@ stimuli: - "Message delete operations use both message_id and pop_receipt from received messages." - "Peek operations demonstrate reading messages without consuming them from the queue." - "Avoids hardcoded credentials in source code." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -83,7 +82,7 @@ stimuli: - type: prompt name: "Queue operations completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - QueueServiceClient initialization with endpoint URL and credential - QueueClient derived from service client @@ -159,7 +158,7 @@ stimuli: - name: QueueServiceClient Construction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > QueueServiceClient is created via QueueServiceClient::new() with three parameters: service_url (Url type), Some(credential), and None for options. @@ -167,7 +166,7 @@ stimuli: - name: Queue Message Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Send messages via queue_client.send_message(message.try_into()?, None).await?. Receive messages via queue_client.receive_messages(None).await? returning a response @@ -177,7 +176,7 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Queue operations are wrapped in Result-based error handling using ? operator. No .unwrap() calls except where failure is impossible. diff --git a/tests/scenarios/fastapi-router-py/scenarios.yaml b/tests/scenarios/fastapi-router-py/scenarios.yaml index d4fa0ae5..8d9efbe2 100644 --- a/tests/scenarios/fastapi-router-py/scenarios.yaml +++ b/tests/scenarios/fastapi-router-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific FastAPI router pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 @@ -36,16 +36,16 @@ scenarios: mock_response: | from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query, status - + from app.models.project import ProjectResponse, ProjectCreate, ProjectUpdate from app.services.project_service import ProjectService - + router = APIRouter(prefix="/api", tags=["projects"]) - + def get_service() -> ProjectService: """Dependency to get service instance.""" return ProjectService() - + @router.get("/projects", response_model=list[ProjectResponse]) async def list_projects( limit: int = Query(default=50, ge=1, le=100), @@ -54,7 +54,7 @@ scenarios: ) -> list[ProjectResponse]: """List all projects with pagination.""" return await service.list_projects(limit=limit, offset=offset) - + @router.get("/projects/{project_id}", response_model=ProjectResponse) async def get_project( project_id: str, @@ -68,7 +68,7 @@ scenarios: detail="Project not found", ) return result - + @router.post( "/projects", response_model=ProjectResponse, @@ -80,7 +80,7 @@ scenarios: ) -> ProjectResponse: """Create a new project.""" return await service.create_project(data) - + @router.patch("/projects/{project_id}", response_model=ProjectResponse) async def update_project( project_id: str, @@ -95,7 +95,7 @@ scenarios: detail="Project not found", ) return await service.update_project(project_id, data) - + @router.delete( "/projects/{project_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -130,27 +130,27 @@ scenarios: - "HTTP_401_UNAUTHORIZED" - "HTTP_403_FORBIDDEN" forbidden_patterns: - - "current_user: str" # Should be User type - - "current_user: Optional[^\\[]" # Should have full type like Optional[User] - - "get_current_user_required.*Optional" # Can't be both required and optional + - "current_user: str" # Should be User type + - "current_user: Optional[^\\[]" # Should have full type like Optional[User] + - "get_current_user_required.*Optional" # Can't be both required and optional tags: - authentication - security mock_response: | from typing import Optional from fastapi import APIRouter, Depends, HTTPException, status - + from app.auth.jwt import get_current_user, get_current_user_required from app.models.user import User from app.models.task import TaskResponse, TaskCreate, TaskUpdate from app.services.task_service import TaskService - + router = APIRouter(prefix="/api", tags=["tasks"]) - + def get_service() -> TaskService: """Dependency to get service instance.""" return TaskService() - + @router.get("/tasks", response_model=list[TaskResponse]) async def list_tasks( current_user: Optional[User] = Depends(get_current_user), @@ -159,7 +159,7 @@ scenarios: """List tasks. Authentication optional.""" user_id = current_user.id if current_user else None return await service.list_tasks(user_id=user_id) - + @router.post( "/tasks", response_model=TaskResponse, @@ -172,7 +172,7 @@ scenarios: ) -> TaskResponse: """Create a new task. Authentication required.""" return await service.create_task(data, owner_id=current_user.id) - + @router.patch("/tasks/{task_id}", response_model=TaskResponse) async def update_task( task_id: str, @@ -193,7 +193,7 @@ scenarios: detail="Not authorized to update this task", ) return await service.update_task(task_id, data) - + @router.delete( "/tasks/{task_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -237,7 +237,7 @@ scenarios: forbidden_patterns: - "response_model=dict" - "response_model=str" - - "response_model=list[^\\[]" # Should specify type: list[Model], not just list + - "response_model=list[^\\[]" # Should specify type: list[Model], not just list - "-> dict:" - "-> Any:" tags: @@ -246,22 +246,22 @@ scenarios: mock_response: | from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel - + from app.models.article import ArticleResponse, ArticleCreate from app.services.article_service import ArticleService - + router = APIRouter(prefix="/api", tags=["articles"]) - + def get_service() -> ArticleService: return ArticleService() - + @router.get("/articles", response_model=list[ArticleResponse]) async def list_articles( service: ArticleService = Depends(get_service), ) -> list[ArticleResponse]: """List all articles.""" return await service.list_articles() - + @router.get("/articles/{article_id}", response_model=ArticleResponse) async def get_article( article_id: str, @@ -275,7 +275,7 @@ scenarios: detail="Article not found", ) return article - + @router.post( "/articles", response_model=ArticleResponse, @@ -287,7 +287,7 @@ scenarios: ) -> ArticleResponse: """Create a new article.""" return await service.create_article(data) - + @router.delete( "/articles/{article_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -323,42 +323,52 @@ scenarios: - "Query\\(default=" - "description=" forbidden_patterns: - - "q: str" # Should have Query validation - - "limit: int = 50" # Should have Query with constraints - - "offset: int = 0" # Should have Query with constraints - - "Query\\(\\)" # Query should have parameters + - "limit: int = 50" # Should have Query with constraints + - "offset: int = 0" # Should have Query with constraints + - "Query\\(\\)" # Query should have parameters tags: - validation - query-parameters mock_response: | - from typing import Annotated, Optional - from fastapi import APIRouter, Depends, Query, status - - from app.models.search import SearchResponse - from app.services.search_service import SearchService - - router = APIRouter(prefix="/api", tags=["search"]) - - def get_service() -> SearchService: - return SearchService() - - @router.get("/search", response_model=list[SearchResponse]) + from typing import Literal + + from fastapi import APIRouter, Query + + router = APIRouter(prefix="/search", tags=["search"]) + + + @router.get("") async def search( - q: Annotated[str, Query(min_length=1, max_length=200, description="Search query")], - limit: Annotated[int, Query(ge=1, le=100, description="Max results")] = 50, - offset: Annotated[int, Query(ge=0, description="Results to skip")] = 0, - sort_by: Annotated[Optional[str], Query(description="Sort by: relevance, date, popularity")] = "relevance", - service: SearchService = Depends(get_service), - ) -> list[SearchResponse]: - """ - Search for items. - - - **q**: Search term (required, 1-200 characters) - - **limit**: Maximum results to return (1-100, default 50) - - **offset**: Number of results to skip (default 0) - - **sort_by**: Sort results by relevance, date, or popularity - """ - return await service.search(q=q, limit=limit, offset=offset, sort_by=sort_by) + q: str = Query( + ..., + min_length=1, + max_length=200, + description="Search term to query for.", + ), + limit: int = Query( + 50, + ge=1, + le=100, + description="Maximum number of results to return.", + ), + offset: int = Query( + 0, + ge=0, + description="Number of results to skip before returning matches.", + ), + sort_by: Literal["relevance", "date", "popularity"] = Query( + "relevance", + description="Sort order for search results.", + ), + ) -> dict[str, object]: + """Search for items matching the provided query parameters.""" + return { + "q": q, + "limit": limit, + "offset": offset, + "sort_by": sort_by, + "results": [], + } # Error Handling with HTTPException - name: error_handling_http_exception @@ -377,25 +387,25 @@ scenarios: - "if.*is None" - "if.*owner_id" forbidden_patterns: - - "except" # Should use HTTPException, not try/except - - "return None" # Should raise HTTPException instead - - "raise Exception" # Should use HTTPException + - "except" # Should use HTTPException, not try/except + - "return None" # Should raise HTTPException instead + - "raise Exception" # Should use HTTPException tags: - error-handling - security mock_response: | from fastapi import APIRouter, Depends, HTTPException, status - + from app.models.user import UserResponse, UserUpdate from app.auth.jwt import get_current_user_required from app.models.user import User from app.services.user_service import UserService - + router = APIRouter(prefix="/api", tags=["users"]) - + def get_service() -> UserService: return UserService() - + @router.get("/users/{user_id}", response_model=UserResponse) async def get_user( user_id: str, @@ -409,7 +419,7 @@ scenarios: detail="User not found", ) return user - + @router.patch("/users/{user_id}", response_model=UserResponse) async def update_user( user_id: str, @@ -456,28 +466,28 @@ scenarios: - "posts" - "comments" forbidden_patterns: - - "include_router.*prefix.*prefix" # Double prefix mounting - - "APIRouter\\(\\)" # Should have prefix + - "include_router.*prefix.*prefix" # Double prefix mounting + - "APIRouter\\(\\)" # Should have prefix tags: - routing - advanced mock_response: | # routers/posts.py from fastapi import APIRouter, Depends, HTTPException, status - + from app.models.post import PostResponse, PostCreate from app.services.post_service import PostService - + router = APIRouter(prefix="/api/posts", tags=["posts"]) - + def get_service() -> PostService: return PostService() - + @router.get("", response_model=list[PostResponse]) async def list_posts(service: PostService = Depends(get_service)) -> list[PostResponse]: """List all posts.""" return await service.list_posts() - + @router.post( "", response_model=PostResponse, @@ -489,23 +499,23 @@ scenarios: ) -> PostResponse: """Create a new post.""" return await service.create_post(data) - + # routers/comments.py from fastapi import APIRouter, Depends, HTTPException, status - + from app.models.comment import CommentResponse, CommentCreate from app.services.comment_service import CommentService - + router = APIRouter(prefix="/api/comments", tags=["comments"]) - + def get_service() -> CommentService: return CommentService() - + @router.get("", response_model=list[CommentResponse]) async def list_comments(service: CommentService = Depends(get_service)) -> list[CommentResponse]: """List all comments.""" return await service.list_comments() - + @router.post( "", response_model=CommentResponse, @@ -517,12 +527,12 @@ scenarios: ) -> CommentResponse: """Create a new comment.""" return await service.create_comment(data) - + # main.py from fastapi import FastAPI from routers.posts import router as posts_router from routers.comments import router as comments_router - + app = FastAPI(title="My API") app.include_router(posts_router) app.include_router(comments_router) @@ -543,32 +553,32 @@ scenarios: - "NotificationService" - "HTTPException" forbidden_patterns: - - "(? NotificationService: """Dependency to get service instance.""" return NotificationService() - + @router.get("/notifications", response_model=list[NotificationResponse]) async def list_notifications( service: NotificationService = Depends(get_service), ) -> list[NotificationResponse]: """List all notifications.""" return await service.list_notifications() - + @router.get("/notifications/{notification_id}", response_model=NotificationResponse) async def get_notification( notification_id: str, @@ -582,7 +592,7 @@ scenarios: detail="Notification not found", ) return notification - + @router.post( "/notifications", response_model=NotificationResponse, @@ -594,7 +604,7 @@ scenarios: ) -> NotificationResponse: """Create a new notification.""" return await service.create_notification(data) - + @router.delete( "/notifications/{notification_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -629,26 +639,26 @@ scenarios: - "service.*=.*Depends" - "current_user.*=.*Depends" forbidden_patterns: - - "service = ReportService\\(\\)" # Should use Depends - - "current_user: str" # Should be User type + - "service = ReportService\\(\\)" # Should use Depends + - "current_user: str" # Should be User type tags: - dependencies - advanced mock_response: | from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query, status - + from app.auth.jwt import get_current_user_required from app.models.user import User from app.models.report import ReportResponse, ReportCreate from app.services.report_service import ReportService - + router = APIRouter(prefix="/api", tags=["reports"]) - + def get_service() -> ReportService: """Dependency to get service instance.""" return ReportService() - + @router.get("/reports", response_model=list[ReportResponse]) async def list_reports( limit: int = Query(default=50, ge=1, le=100), @@ -664,7 +674,7 @@ scenarios: offset=offset, status_filter=status_filter, ) - + @router.post( "/reports", response_model=ReportResponse, @@ -677,7 +687,7 @@ scenarios: ) -> ReportResponse: """Create a new report for the current user.""" return await service.create_report(data, user_id=current_user.id) - + @router.get("/reports/{report_id}", response_model=ReportResponse) async def get_report( report_id: str, diff --git a/tests/scenarios/fastapi-router-py/vally/eval.yaml b/tests/scenarios/fastapi-router-py/vally/eval.yaml new file mode 100644 index 00000000..a5c3a5ff --- /dev/null +++ b/tests/scenarios/fastapi-router-py/vally/eval.yaml @@ -0,0 +1,159 @@ +name: fastapi-router-py-eval +description: Vally evaluation suite for fastapi-router-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_crud_router + prompt: |- + Create a FastAPI router for managing "projects" with full CRUD operations. + Include GET (list and single), POST (create), PATCH (update), and DELETE endpoints. + Use proper status codes, response models, and async/await patterns. + Include pagination with limit and offset query parameters. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "project" (snake_case), "Project" (PascalCase), "projects" (plural) + * NEVER use {{variable}}, {{resource_name}}, {{ResourceName}}, or {{resource_plural}} + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling: + * Wrap async endpoint logic in try/except blocks + * Handle HTTPException, ValueError, and generic Exception types + * Provide meaningful error messages in responses + * Example: try: ... except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: router_with_authentication + prompt: |- + Create a FastAPI router for "tasks" where: + - GET endpoints (list, single) allow optional authentication + - POST, PATCH, DELETE endpoints require authentication + - Use get_current_user for optional and get_current_user_required for required auth + Include proper type hints and HTTPException for unauthorized access. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "task" (snake_case), "Task" (PascalCase), "tasks" (plural) + * NEVER use {{variable}}, {{resource_name}}, {{ResourceName}}, or {{resource_plural}} + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling: + * Wrap async endpoint logic in try/except blocks + * Handle HTTPException, ValueError, and generic Exception types + * Provide meaningful error messages in responses + * Example: try: ... except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - fastapi-router-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/fastapi-router-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..a5c3a5ff --- /dev/null +++ b/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,159 @@ +name: fastapi-router-py-eval +description: Vally evaluation suite for fastapi-router-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_crud_router + prompt: |- + Create a FastAPI router for managing "projects" with full CRUD operations. + Include GET (list and single), POST (create), PATCH (update), and DELETE endpoints. + Use proper status codes, response models, and async/await patterns. + Include pagination with limit and offset query parameters. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "project" (snake_case), "Project" (PascalCase), "projects" (plural) + * NEVER use {{variable}}, {{resource_name}}, {{ResourceName}}, or {{resource_plural}} + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling: + * Wrap async endpoint logic in try/except blocks + * Handle HTTPException, ValueError, and generic Exception types + * Provide meaningful error messages in responses + * Example: try: ... except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: router_with_authentication + prompt: |- + Create a FastAPI router for "tasks" where: + - GET endpoints (list, single) allow optional authentication + - POST, PATCH, DELETE endpoints require authentication + - Use get_current_user for optional and get_current_user_required for required auth + Include proper type hints and HTTPException for unauthorized access. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "task" (snake_case), "Task" (PascalCase), "tasks" (plural) + * NEVER use {{variable}}, {{resource_name}}, {{ResourceName}}, or {{resource_plural}} + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling: + * Wrap async endpoint logic in try/except blocks + * Handle HTTPException, ValueError, and generic Exception types + * Provide meaningful error messages in responses + * Example: try: ... except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - fastapi-router-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/fastapi-router-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..abf441ec --- /dev/null +++ b/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: fastapi-router-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/fastapi-router-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/fastapi-router-py/vally/syntax-check-config.json b/tests/scenarios/fastapi-router-py/vally/syntax-check-config.json new file mode 100644 index 00000000..4b941664 --- /dev/null +++ b/tests/scenarios/fastapi-router-py/vally/syntax-check-config.json @@ -0,0 +1,6 @@ +{ + "excludePatterns": [ + "*/assets/template.py", + "**/_**/template.py" + ] +} \ No newline at end of file diff --git a/tests/scenarios/m365-agents-py/scenarios.yaml b/tests/scenarios/m365-agents-py/scenarios.yaml index 40d0aaa1..25e9e4b3 100644 --- a/tests/scenarios/m365-agents-py/scenarios.yaml +++ b/tests/scenarios/m365-agents-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/m365-agents-py/vally/eval.yaml b/tests/scenarios/m365-agents-py/vally/eval.yaml new file mode 100644 index 00000000..a434d054 --- /dev/null +++ b/tests/scenarios/m365-agents-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: m365-agents-py-eval +description: Vally evaluation suite for m365-agents-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: aiohttp_agent_hosting + prompt: |- + Create a minimal aiohttp-hosted Microsoft 365 agent in Python using + AgentApplication, start_agent_process, and jwt_authorization_middleware. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap AgentApplication initialization and turn/message handling in try/except blocks + * Handle: AuthenticationError, ServiceError, generic Exception + * Example: try: await app.process_activity(req) except Exception as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: agent_application_routing + prompt: |- + Create an AgentApplication that handles welcome messages with + conversation_update, message handlers with regex, and an error handler. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap AgentApplication initialization and turn/message handling in try/except blocks + * Handle: AuthenticationError, ServiceError, generic Exception + * Example: try: await app.process_activity(req) except Exception as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - m365-agents-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/m365-agents-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/m365-agents-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/m365-agents-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..a434d054 --- /dev/null +++ b/tests/scenarios/m365-agents-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: m365-agents-py-eval +description: Vally evaluation suite for m365-agents-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: aiohttp_agent_hosting + prompt: |- + Create a minimal aiohttp-hosted Microsoft 365 agent in Python using + AgentApplication, start_agent_process, and jwt_authorization_middleware. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap AgentApplication initialization and turn/message handling in try/except blocks + * Handle: AuthenticationError, ServiceError, generic Exception + * Example: try: await app.process_activity(req) except Exception as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: agent_application_routing + prompt: |- + Create an AgentApplication that handles welcome messages with + conversation_update, message handlers with regex, and an error handler. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap AgentApplication initialization and turn/message handling in try/except blocks + * Handle: AuthenticationError, ServiceError, generic Exception + * Example: try: await app.process_activity(req) except Exception as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - m365-agents-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/m365-agents-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/m365-agents-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/m365-agents-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..bb656e5c --- /dev/null +++ b/tests/scenarios/m365-agents-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: m365-agents-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/m365-agents-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/pydantic-models-py/scenarios.yaml b/tests/scenarios/pydantic-models-py/scenarios.yaml index 0a10ccb2..1af0cde2 100644 --- a/tests/scenarios/pydantic-models-py/scenarios.yaml +++ b/tests/scenarios/pydantic-models-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific Pydantic model pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/pydantic-models-py/vally/eval.yaml b/tests/scenarios/pydantic-models-py/vally/eval.yaml new file mode 100644 index 00000000..d35001a5 --- /dev/null +++ b/tests/scenarios/pydantic-models-py/vally/eval.yaml @@ -0,0 +1,156 @@ +name: pydantic-models-py-eval +description: Vally evaluation suite for pydantic-models-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: base_create_response_indb + prompt: |- + Create a complete set of Pydantic models for a "Document" resource following the + multi-model pattern. Include Base, Create, Update, Response, and InDB models. + Use Field constraints, proper inheritance, and camelCase aliases with populate_by_name. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "Document" (PascalCase), "document" (snake_case) + * NEVER use {{variable}}, {{ResourceName}}, {{resource_name}}, or similar placeholders + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling in any code that instantiates or validates models: + * Wrap model instantiation in try/except blocks + * Handle ValidationError from pydantic + * Example: try: Document(...) except ValidationError as e: handle_validation_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: camel_case_aliases + prompt: |- + Create a Pydantic model for a "Project" resource with snake_case fields but camelCase + aliases for API responses. Include created_at and updated_at with aliases. + Ensure populate_by_name is True to accept both formats. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "Project" (PascalCase), "project" (snake_case) + * NEVER use {{variable}}, {{ResourceName}}, {{resource_name}}, or similar placeholders + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling in any code that instantiates or validates models: + * Wrap model instantiation in try/except blocks + * Handle ValidationError from pydantic + * Example: try: Project(...) except ValidationError as e: handle_validation_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - pydantic-models-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/pydantic-models-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d35001a5 --- /dev/null +++ b/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,156 @@ +name: pydantic-models-py-eval +description: Vally evaluation suite for pydantic-models-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: base_create_response_indb + prompt: |- + Create a complete set of Pydantic models for a "Document" resource following the + multi-model pattern. Include Base, Create, Update, Response, and InDB models. + Use Field constraints, proper inheritance, and camelCase aliases with populate_by_name. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "Document" (PascalCase), "document" (snake_case) + * NEVER use {{variable}}, {{ResourceName}}, {{resource_name}}, or similar placeholders + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling in any code that instantiates or validates models: + * Wrap model instantiation in try/except blocks + * Handle ValidationError from pydantic + * Example: try: Document(...) except ValidationError as e: handle_validation_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: camel_case_aliases + prompt: |- + Create a Pydantic model for a "Project" resource with snake_case fields but camelCase + aliases for API responses. Include created_at and updated_at with aliases. + Ensure populate_by_name is True to accept both formats. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "Project" (PascalCase), "project" (snake_case) + * NEVER use {{variable}}, {{ResourceName}}, {{resource_name}}, or similar placeholders + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling in any code that instantiates or validates models: + * Wrap model instantiation in try/except blocks + * Handle ValidationError from pydantic + * Example: try: Project(...) except ValidationError as e: handle_validation_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - pydantic-models-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/pydantic-models-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..9f89784e --- /dev/null +++ b/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: pydantic-models-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/pydantic-models-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/pydantic-models-py/vally/syntax-check-config.json b/tests/scenarios/pydantic-models-py/vally/syntax-check-config.json new file mode 100644 index 00000000..4b941664 --- /dev/null +++ b/tests/scenarios/pydantic-models-py/vally/syntax-check-config.json @@ -0,0 +1,6 @@ +{ + "excludePatterns": [ + "*/assets/template.py", + "**/_**/template.py" + ] +} \ No newline at end of file