From 3d04818b15e422a8f370543d624cf9981671c767 Mon Sep 17 00:00:00 2001 From: Sam Betts <1769706+Tehsmash@users.noreply.github.com> Date: Tue, 19 May 2026 11:00:40 +0100 Subject: [PATCH 1/2] feat(slimctl): add Claude Code agent skill Add a Claude Code skill for slimctl under skills/slimctl/SKILL.md. The skill provides a concise command reference, route format docs, global flag reference, and common workflows (mTLS setup, route add, channel management). Includes an explicit warning that tls-insecure defaults to true and must be set to false when using TLS certificates. The release workflow is extended with a publish-skill job that packages the skill directory into a slimctl.skill zip artifact and uploads it to the GitHub release alongside the binary tarballs. Refs: #slimctl-agent-skill Signed-off-by: Sam Betts <1769706+Tehsmash@users.noreply.github.com> --- .github/workflows/release-slimctl.yaml | 39 +++ .../iteration-1/benchmark.json | 275 ++++++++++++++++++ .../iteration-1/benchmark.md | 13 + .../iteration-1/eval-1/eval_metadata.json | 32 ++ .../eval-1/with_skill/grading.json | 31 ++ .../eval-1/with_skill/outputs/response.md | 42 +++ .../iteration-1/eval-1/with_skill/timing.json | 5 + .../eval-1/without_skill/grading.json | 31 ++ .../eval-1/without_skill/outputs/response.md | 69 +++++ .../eval-1/without_skill/timing.json | 5 + .../iteration-1/eval-2/eval_metadata.json | 37 +++ .../eval-2/with_skill/grading.json | 36 +++ .../eval-2/with_skill/outputs/response.md | 28 ++ .../iteration-1/eval-2/with_skill/timing.json | 5 + .../eval-2/without_skill/grading.json | 36 +++ .../eval-2/without_skill/outputs/response.md | 39 +++ .../eval-2/without_skill/timing.json | 5 + .../iteration-1/eval-3/eval_metadata.json | 32 ++ .../eval-3/with_skill/grading.json | 31 ++ .../eval-3/with_skill/outputs/response.md | 45 +++ .../iteration-1/eval-3/with_skill/timing.json | 5 + .../eval-3/without_skill/grading.json | 31 ++ .../eval-3/without_skill/outputs/response.md | 39 +++ .../eval-3/without_skill/timing.json | 5 + .../iteration-1/eval-4/eval_metadata.json | 32 ++ .../eval-4/with_skill/grading.json | 31 ++ .../eval-4/with_skill/outputs/response.md | 45 +++ .../iteration-1/eval-4/with_skill/timing.json | 5 + .../eval-4/without_skill/grading.json | 31 ++ .../eval-4/without_skill/outputs/response.md | 93 ++++++ .../eval-4/without_skill/timing.json | 5 + .../iteration-1/eval-5/eval_metadata.json | 22 ++ .../eval-5/with_skill/grading.json | 21 ++ .../eval-5/with_skill/outputs/response.md | 29 ++ .../iteration-1/eval-5/with_skill/timing.json | 5 + .../eval-5/without_skill/grading.json | 21 ++ .../eval-5/without_skill/outputs/response.md | 78 +++++ .../eval-5/without_skill/timing.json | 5 + .../iteration-1/feedback.json | 15 + .../iteration-2/eval-3/eval_metadata.json | 32 ++ .../eval-3/with_skill/grading.json | 31 ++ .../eval-3/with_skill/outputs/response.md | 21 ++ .../iteration-2/eval-3/with_skill/timing.json | 5 + skills/slimctl-workspace/trigger-evals.json | 82 ++++++ skills/slimctl/SKILL.md | 214 ++++++++++++++ skills/slimctl/evals/evals.json | 35 +++ 46 files changed, 1774 insertions(+) create mode 100644 skills/slimctl-workspace/iteration-1/benchmark.json create mode 100644 skills/slimctl-workspace/iteration-1/benchmark.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-1/eval_metadata.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-1/with_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-1/with_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-1/with_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-1/without_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-1/without_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-1/without_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-2/eval_metadata.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-2/with_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-2/with_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-2/with_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-2/without_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-2/without_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-2/without_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-3/eval_metadata.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-3/with_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-3/with_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-3/with_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-3/without_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-3/without_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-3/without_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-4/eval_metadata.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-4/with_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-4/with_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-4/with_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-4/without_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-4/without_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-4/without_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-5/eval_metadata.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-5/with_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-5/with_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-5/with_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-5/without_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-1/eval-5/without_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-1/eval-5/without_skill/timing.json create mode 100644 skills/slimctl-workspace/iteration-1/feedback.json create mode 100644 skills/slimctl-workspace/iteration-2/eval-3/eval_metadata.json create mode 100644 skills/slimctl-workspace/iteration-2/eval-3/with_skill/grading.json create mode 100644 skills/slimctl-workspace/iteration-2/eval-3/with_skill/outputs/response.md create mode 100644 skills/slimctl-workspace/iteration-2/eval-3/with_skill/timing.json create mode 100644 skills/slimctl-workspace/trigger-evals.json create mode 100644 skills/slimctl/SKILL.md create mode 100644 skills/slimctl/evals/evals.json diff --git a/.github/workflows/release-slimctl.yaml b/.github/workflows/release-slimctl.yaml index 5604d6d59..3b68d54ec 100644 --- a/.github/workflows/release-slimctl.yaml +++ b/.github/workflows/release-slimctl.yaml @@ -96,6 +96,45 @@ jobs: manifest-path: ./data-plane/slimctl/Cargo.toml build-tool: ${{ matrix.platform.build-tool }} + publish-skill: + runs-on: ubuntu-latest + needs: [ensure-release] + permissions: + contents: write + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Package slimctl skill + run: | + python3 - <<'EOF' + import zipfile, pathlib + + skill_dir = pathlib.Path("skills/slimctl") + out = pathlib.Path("slimctl.skill") + exclude_dirs = {"evals", "__pycache__"} + + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zf: + for f in sorted(skill_dir.rglob("*")): + if not f.is_file(): + continue + rel = f.relative_to(skill_dir.parent) + if any(part in exclude_dirs for part in rel.parts): + continue + zf.write(f, rel) + print(f" added: {rel}") + + print(f"created {out} ({out.stat().st_size} bytes)") + EOF + + - name: Upload skill to release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "${{ github.ref_name }}" slimctl.skill --clobber + homebrew: runs-on: ubuntu-latest needs: [build-binaries] diff --git a/skills/slimctl-workspace/iteration-1/benchmark.json b/skills/slimctl-workspace/iteration-1/benchmark.json new file mode 100644 index 000000000..35a18a7e1 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/benchmark.json @@ -0,0 +1,275 @@ +{ + "metadata": { + "skill_name": "slimctl", + "skill_path": "skills/slimctl", + "executor_model": "claude-sonnet-4-6", + "analyzer_model": "claude-sonnet-4-6", + "timestamp": "2026-05-18T12:49:17Z", + "evals_run": [1, 2, 3, 4, 5], + "runs_per_configuration": 1 + }, + "runs": [ + { + "eval_id": 1, + "eval_name": "node-route-add", + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 5, + "failed": 0, + "total": 5, + "time_seconds": 19.5, + "tokens": 14698, + "tool_calls": 3, + "errors": 0 + }, + "expectations": [ + {"text": "Uses 'node route add' subcommand (not controller)", "passed": true, "evidence": "slimctl node route add acme/production/assistant/1 via /tmp/conn.json"}, + {"text": "Route is in org/ns/agent/id format: acme/production/assistant/1", "passed": true, "evidence": "Route breakdown shown in notes"}, + {"text": "Includes the 'via' keyword", "passed": true, "evidence": "node route add ... via /tmp/conn.json"}, + {"text": "References a JSON connection config file (not inline URL)", "passed": true, "evidence": "via /tmp/conn.json"}, + {"text": "Shows or mentions the JSON file should contain the endpoint http://10.0.0.5:8080", "passed": true, "evidence": "{ \"endpoint\": \"http://10.0.0.5:8080\" }"} + ], + "notes": [] + }, + { + "eval_id": 1, + "eval_name": "node-route-add", + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 5, + "failed": 0, + "total": 5, + "time_seconds": 58.7, + "tokens": 42095, + "tool_calls": 13, + "errors": 0 + }, + "expectations": [ + {"text": "Uses 'node route add' subcommand (not controller)", "passed": true, "evidence": "Found correct command by reading source code"}, + {"text": "Route is in org/ns/agent/id format: acme/production/assistant/1", "passed": true, "evidence": "four slash-separated components"}, + {"text": "Includes the 'via' keyword", "passed": true, "evidence": "node route add ... via /tmp/conn.json"}, + {"text": "References a JSON connection config file (not inline URL)", "passed": true, "evidence": "via /tmp/conn.json"}, + {"text": "Shows or mentions the JSON file should contain the endpoint http://10.0.0.5:8080", "passed": true, "evidence": "{ \"endpoint\": \"http://10.0.0.5:8080\" }"} + ], + "notes": ["Agent read source code from data-plane/slimctl/src/ to derive the answer — 13 tool calls vs 3 for with_skill"] + }, + { + "eval_id": 2, + "eval_name": "controller-route-list-and-del", + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 6, + "failed": 0, + "total": 6, + "time_seconds": 21.4, + "tokens": 14689, + "tool_calls": 3, + "errors": 0 + }, + "expectations": [ + {"text": "Uses 'controller route list' (not 'node route list')", "passed": true, "evidence": "slimctl controller route list -n edge-node-1"}, + {"text": "Passes -n edge-node-1 to the list command", "passed": true, "evidence": "slimctl controller route list -n edge-node-1"}, + {"text": "Uses 'controller route del' for deletion", "passed": true, "evidence": "slimctl controller route del -n edge-node-1 acme/prod/chatbot/3 via http://gateway:9090"}, + {"text": "Deletion command includes -n edge-node-1", "passed": true, "evidence": "slimctl controller route del -n edge-node-1 ..."}, + {"text": "Deletion command includes route acme/prod/chatbot/3", "passed": true, "evidence": "... acme/prod/chatbot/3 via http://gateway:9090"}, + {"text": "Deletion command includes 'via http://gateway:9090'", "passed": true, "evidence": "... via http://gateway:9090"} + ], + "notes": [] + }, + { + "eval_id": 2, + "eval_name": "controller-route-list-and-del", + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 6, + "failed": 0, + "total": 6, + "time_seconds": 38.5, + "tokens": 17524, + "tool_calls": 11, + "errors": 0 + }, + "expectations": [ + {"text": "Uses 'controller route list' (not 'node route list')", "passed": true, "evidence": "slimctl controller route list -n edge-node-1"}, + {"text": "Passes -n edge-node-1 to the list command", "passed": true, "evidence": "slimctl controller route list -n edge-node-1"}, + {"text": "Uses 'controller route del' for deletion", "passed": true, "evidence": "slimctl controller route del -n edge-node-1 acme/prod/chatbot/3 via http://gateway:9090"}, + {"text": "Deletion command includes -n edge-node-1", "passed": true, "evidence": "slimctl controller route del -n edge-node-1 ..."}, + {"text": "Deletion command includes route acme/prod/chatbot/3", "passed": true, "evidence": "... acme/prod/chatbot/3 via http://gateway:9090"}, + {"text": "Deletion command includes 'via http://gateway:9090'", "passed": true, "evidence": "... via http://gateway:9090"} + ], + "notes": ["Used 11 tool calls vs 3 — searched the codebase"] + }, + { + "eval_id": 3, + "eval_name": "configure-tls", + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 0.8, + "passed": 4, + "failed": 1, + "total": 5, + "time_seconds": 24.5, + "tokens": 14930, + "tool_calls": 3, + "errors": 0 + }, + "expectations": [ + {"text": "Sets server to myhost.internal:9090 via 'config set server'", "passed": true, "evidence": "slimctl config set server myhost.internal:9090"}, + {"text": "Sets tls-ca-file to /etc/pki/ca.pem", "passed": true, "evidence": "slimctl config set tls-ca-file /etc/pki/ca.pem"}, + {"text": "Sets tls-cert-file to /etc/pki/client.pem", "passed": true, "evidence": "slimctl config set tls-cert-file /etc/pki/client.pem"}, + {"text": "Sets tls-key-file to /etc/pki/client.key", "passed": true, "evidence": "slimctl config set tls-key-file /etc/pki/client.key"}, + {"text": "Disables tls-insecure (sets to false) since real TLS is being configured", "passed": false, "evidence": "Response incorrectly states 'no need to set tls-insecure to false' — but tls-insecure defaults to true, so TLS will NOT be used without this command"} + ], + "notes": ["CRITICAL: Skill does not document that tls-insecure defaults to true. With-skill response gave actively wrong advice saying TLS is implicitly enabled when cert files are provided."] + }, + { + "eval_id": 3, + "eval_name": "configure-tls", + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 5, + "failed": 0, + "total": 5, + "time_seconds": 41.3, + "tokens": 24353, + "tool_calls": 12, + "errors": 0 + }, + "expectations": [ + {"text": "Sets server to myhost.internal:9090 via 'config set server'", "passed": true, "evidence": "slimctl config set server myhost.internal:9090"}, + {"text": "Sets tls-ca-file to /etc/pki/ca.pem", "passed": true, "evidence": "slimctl config set tls-ca-file /etc/pki/ca.pem"}, + {"text": "Sets tls-cert-file to /etc/pki/client.pem", "passed": true, "evidence": "slimctl config set tls-cert-file /etc/pki/client.pem"}, + {"text": "Sets tls-key-file to /etc/pki/client.key", "passed": true, "evidence": "slimctl config set tls-key-file /etc/pki/client.key"}, + {"text": "Disables tls-insecure (sets to false) since real TLS is being configured", "passed": true, "evidence": "slimctl config set tls-insecure false — explicitly included. Agent read config.rs and found DEFAULT_TLS_INSECURE = true"} + ], + "notes": ["Agent read source code and found tls-insecure defaults to true in defaults.rs — correctly included tls-insecure false step"] + }, + { + "eval_id": 4, + "eval_name": "channel-and-participants", + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 5, + "failed": 0, + "total": 5, + "time_seconds": 24.5, + "tokens": 14782, + "tool_calls": 3, + "errors": 0 + }, + "expectations": [ + {"text": "Uses 'controller channel create' with moderators=alice", "passed": true, "evidence": "slimctl controller channel create moderators=alice"}, + {"text": "Uses 'controller participant add bob -c '", "passed": true, "evidence": "slimctl controller participant add bob -c "}, + {"text": "Uses 'controller participant add carol -c '", "passed": true, "evidence": "slimctl controller participant add carol -c "}, + {"text": "Uses 'controller participant list -c '", "passed": true, "evidence": "slimctl controller participant list -c "}, + {"text": "Explains that the channel ID comes from the output of the channel create step", "passed": true, "evidence": "The placeholder should be replaced with the actual ID returned by the 'channel create' command"} + ], + "notes": [] + }, + { + "eval_id": 4, + "eval_name": "channel-and-participants", + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 5, + "failed": 0, + "total": 5, + "time_seconds": 60.8, + "tokens": 41229, + "tool_calls": 14, + "errors": 0 + }, + "expectations": [ + {"text": "Uses 'controller channel create' with moderators=alice", "passed": true, "evidence": "slimctl controller channel create moderators=alice"}, + {"text": "Uses 'controller participant add bob -c '", "passed": true, "evidence": "slimctl controller participant add bob -c "}, + {"text": "Uses 'controller participant add carol -c '", "passed": true, "evidence": "slimctl controller participant add carol -c "}, + {"text": "Uses 'controller participant list -c '", "passed": true, "evidence": "slimctl controller participant list -c "}, + {"text": "Explains that the channel ID comes from the output of the channel create step", "passed": true, "evidence": "Agent read controller.rs source code and derived correct workflow"} + ], + "notes": ["14 tool calls — agent read controller.rs source code"] + }, + { + "eval_id": 5, + "eval_name": "list-routes-and-connections", + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 24.2, + "tokens": 14990, + "tool_calls": 4, + "errors": 0 + }, + "expectations": [ + {"text": "Uses 'node route list' (direct node command, not controller)", "passed": true, "evidence": "slimctl node route list"}, + {"text": "Uses 'node connection list' (or alias 'node conn list')", "passed": true, "evidence": "slimctl node connection list"}, + {"text": "Mentions default server 127.0.0.1:46357 or explains the local node assumption", "passed": true, "evidence": "Both commands default to connecting to 127.0.0.1:46357"} + ], + "notes": [] + }, + { + "eval_id": 5, + "eval_name": "list-routes-and-connections", + "configuration": "without_skill", + "run_number": 1, + "result": { + "pass_rate": 1.0, + "passed": 3, + "failed": 0, + "total": 3, + "time_seconds": 41.2, + "tokens": 38167, + "tool_calls": 8, + "errors": 0 + }, + "expectations": [ + {"text": "Uses 'node route list' (direct node command, not controller)", "passed": true, "evidence": "slimctl node route list"}, + {"text": "Uses 'node connection list' (or alias 'node conn list')", "passed": true, "evidence": "slimctl node connection list"}, + {"text": "Mentions default server 127.0.0.1:46357 or explains the local node assumption", "passed": true, "evidence": "Agent read node.rs source code and identified gRPC control API"} + ], + "notes": ["8 tool calls — agent read node.rs and related source code"] + } + ], + "run_summary": { + "with_skill": { + "pass_rate": {"mean": 0.96, "stddev": 0.089, "min": 0.80, "max": 1.0}, + "time_seconds": {"mean": 22.8, "stddev": 2.3, "min": 19.5, "max": 24.5}, + "tokens": {"mean": 14818, "stddev": 127, "min": 14689, "max": 14990} + }, + "without_skill": { + "pass_rate": {"mean": 1.0, "stddev": 0.0, "min": 1.0, "max": 1.0}, + "time_seconds": {"mean": 48.1, "stddev": 10.8, "min": 38.5, "max": 60.8}, + "tokens": {"mean": 32674, "stddev": 10838, "min": 17524, "max": 42095} + }, + "delta": { + "pass_rate": "-0.04", + "time_seconds": "-25.3", + "tokens": "-13856" + } + }, + "notes": [ + "CRITICAL GAP: Skill missing tls-insecure default. With-skill response for eval 3 gave actively wrong advice — said TLS is 'implicitly enabled' when cert files are provided, but tls-insecure defaults to true (plain HTTP/2). Must add explicit warning to skill.", + "Without-skill baseline in this repo is not a clean baseline — agents have access to the source code and SKILL.md file, so they can find answers by reading the code. All 5 without-skill runs searched the codebase.", + "Speed benefit is very clear: with-skill avg 22.8s and 14.8k tokens vs without-skill 48.1s and 32.7k tokens — 2.1x faster, 2.2x fewer tokens.", + "Assertions 1-4 in eval 3 pass in both configurations (non-discriminating for TLS files). Only assertion 5 (tls-insecure false) differentiates them.", + "After fixing the TLS default in the skill, with-skill should achieve 100% pass rate with the speed advantage maintained." + ] +} diff --git a/skills/slimctl-workspace/iteration-1/benchmark.md b/skills/slimctl-workspace/iteration-1/benchmark.md new file mode 100644 index 000000000..79d9a5061 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/benchmark.md @@ -0,0 +1,13 @@ +# Skill Benchmark: slimctl + +**Model**: +**Date**: 2026-05-18T12:49:17Z +**Evals**: (3 runs each per configuration) + +## Summary + +| Metric | Config A | Config B | Delta | +|--------|------------|---------------|-------| +| Pass Rate | 0% ± 0% | 0% ± 0% | +0.00 | +| Time | 0.0s ± 0.0s | 0.0s ± 0.0s | +0.0s | +| Tokens | 0 ± 0 | 0 ± 0 | +0 | \ No newline at end of file diff --git a/skills/slimctl-workspace/iteration-1/eval-1/eval_metadata.json b/skills/slimctl-workspace/iteration-1/eval-1/eval_metadata.json new file mode 100644 index 000000000..93f35f195 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-1/eval_metadata.json @@ -0,0 +1,32 @@ +{ + "eval_id": 1, + "eval_name": "node-route-add", + "prompt": "I need to add a route on my SLIM node for agent acme/production/assistant/1 that goes via the remote node at http://10.0.0.5:8080. How do I do this with slimctl?", + "assertions": [ + { + "text": "Uses 'node route add' subcommand (not controller)", + "passed": null, + "evidence": null + }, + { + "text": "Route is in org/ns/agent/id format: acme/production/assistant/1", + "passed": null, + "evidence": null + }, + { + "text": "Includes the 'via' keyword", + "passed": null, + "evidence": null + }, + { + "text": "References a JSON connection config file (not inline URL)", + "passed": null, + "evidence": null + }, + { + "text": "Shows or mentions the JSON file should contain the endpoint http://10.0.0.5:8080", + "passed": null, + "evidence": null + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-1/with_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-1/with_skill/grading.json new file mode 100644 index 000000000..71b206b38 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-1/with_skill/grading.json @@ -0,0 +1,31 @@ +{ + "eval_id": 1, + "run": "with_skill", + "expectations": [ + { + "text": "Uses 'node route add' subcommand (not controller)", + "passed": true, + "evidence": "slimctl node route add acme/production/assistant/1 via /tmp/conn.json" + }, + { + "text": "Route is in org/ns/agent/id format: acme/production/assistant/1", + "passed": true, + "evidence": "acme/production/assistant/1 — org=acme, namespace=production, agent=assistant, id=1" + }, + { + "text": "Includes the 'via' keyword", + "passed": true, + "evidence": "node route add acme/production/assistant/1 via /tmp/conn.json" + }, + { + "text": "References a JSON connection config file (not inline URL)", + "passed": true, + "evidence": "via /tmp/conn.json — creates cat > /tmp/conn.json" + }, + { + "text": "Shows or mentions the JSON file should contain the endpoint http://10.0.0.5:8080", + "passed": true, + "evidence": "{ \"endpoint\": \"http://10.0.0.5:8080\" }" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-1/with_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-1/with_skill/outputs/response.md new file mode 100644 index 000000000..14b55f73e --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-1/with_skill/outputs/response.md @@ -0,0 +1,42 @@ +## Adding a Route on a SLIM Node with slimctl + +To add a route for agent `acme/production/assistant/1` that goes via the remote node at `http://10.0.0.5:8080`, you need two steps: + +### Step 1: Create a connection config file + +Create a JSON file that specifies the remote node endpoint: + +```bash +cat > /tmp/conn.json <<'EOF' +{ "endpoint": "http://10.0.0.5:8080" } +EOF +``` + +### Step 2: Add the route + +Use the `slimctl node route add` command, providing the agent route and the path to the connection config file: + +```bash +slimctl node route add acme/production/assistant/1 via /tmp/conn.json +``` + +### Notes + +- The route format is `org/namespace/agentname/agentid`, so `acme/production/assistant/1` breaks down as org=`acme`, namespace=`production`, agent name=`assistant`, agent id=`1`. +- By default, slimctl connects to the local SLIM node at `127.0.0.1:46357`. If your node is running on a different address, use the `--server` flag: + +```bash +slimctl --server node route add acme/production/assistant/1 via /tmp/conn.json +``` + +- You can verify the route was added by listing routes: + +```bash +slimctl node route list +``` + +- To remove the route later: + +```bash +slimctl node route del acme/production/assistant/1 via http://10.0.0.5:8080 +``` diff --git a/skills/slimctl-workspace/iteration-1/eval-1/with_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-1/with_skill/timing.json new file mode 100644 index 000000000..5bc91da03 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-1/with_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 14698, + "duration_ms": 19456, + "total_duration_seconds": 19.5 +} diff --git a/skills/slimctl-workspace/iteration-1/eval-1/without_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-1/without_skill/grading.json new file mode 100644 index 000000000..3c09a0471 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-1/without_skill/grading.json @@ -0,0 +1,31 @@ +{ + "eval_id": 1, + "run": "without_skill", + "expectations": [ + { + "text": "Uses 'node route add' subcommand (not controller)", + "passed": true, + "evidence": "slimctl node route add acme/production/assistant/1 via /tmp/conn.json" + }, + { + "text": "Route is in org/ns/agent/id format: acme/production/assistant/1", + "passed": true, + "evidence": "four slash-separated components: org/namespace/agentname/agentid" + }, + { + "text": "Includes the 'via' keyword", + "passed": true, + "evidence": "node route add ... via /tmp/conn.json" + }, + { + "text": "References a JSON connection config file (not inline URL)", + "passed": true, + "evidence": "via /tmp/conn.json — creates cat > /tmp/conn.json" + }, + { + "text": "Shows or mentions the JSON file should contain the endpoint http://10.0.0.5:8080", + "passed": true, + "evidence": "{ \"endpoint\": \"http://10.0.0.5:8080\" }" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-1/without_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-1/without_skill/outputs/response.md new file mode 100644 index 000000000..e16562646 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-1/without_skill/outputs/response.md @@ -0,0 +1,69 @@ +# Adding a Route with slimctl + +To add a route for agent `acme/production/assistant/1` that goes via a remote node at `http://10.0.0.5:8080`, you need to use the `slimctl node route add` command. This command requires a JSON connection config file that specifies the remote endpoint. + +## Steps + +### 1. Create a connection config file + +Create a JSON file that contains the endpoint of the remote node: + +```bash +cat > /tmp/conn.json <<'EOF' +{ "endpoint": "http://10.0.0.5:8080" } +EOF +``` + +### 2. Add the route + +```bash +slimctl node route add acme/production/assistant/1 via /tmp/conn.json +``` + +This tells your local SLIM node to route messages destined for agent `acme/production/assistant/1` through the remote node at `http://10.0.0.5:8080`. + +## Syntax breakdown + +The `node route add` command syntax is: + +``` +slimctl node route add via +``` + +- The route is specified as four slash-separated components: `org/namespace/agentname/agentid` +- The keyword `via` is a literal separator +- The last argument is a path to a JSON file containing at minimum an `"endpoint"` field + +## Pointing slimctl at a specific node + +By default, slimctl connects to `127.0.0.1:46357`. If your local SLIM node is running on a different address, use the `--server` flag or configure it persistently: + +```bash +# One-off override +slimctl --server localhost:46357 node route add acme/production/assistant/1 via /tmp/conn.json + +# Or persist the setting +slimctl config set server localhost:46357 +``` + +## TLS considerations + +If your node's control API does not use TLS (common in development), slimctl defaults to insecure (plain HTTP/2) mode. If TLS is enabled, you may need flags like `--tls.insecure_skip_verify` or provide CA/cert/key files via `--tls.ca_file`, `--tls.cert_file`, and `--tls.key_file`. + +## Verifying the route + +After adding the route, you can verify it was applied: + +```bash +slimctl node route list +``` + +## Removing the route later + +To remove the route: + +```bash +slimctl node route del acme/production/assistant/1 via http://10.0.0.5:8080 +``` + +Note that for deletion, you specify the endpoint URL directly (not a config file). diff --git a/skills/slimctl-workspace/iteration-1/eval-1/without_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-1/without_skill/timing.json new file mode 100644 index 000000000..7febbe7df --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-1/without_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 42095, + "duration_ms": 58682, + "total_duration_seconds": 58.7 +} diff --git a/skills/slimctl-workspace/iteration-1/eval-2/eval_metadata.json b/skills/slimctl-workspace/iteration-1/eval-2/eval_metadata.json new file mode 100644 index 000000000..b4214565d --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-2/eval_metadata.json @@ -0,0 +1,37 @@ +{ + "eval_id": 2, + "eval_name": "controller-route-list-and-del", + "prompt": "List all routes the control plane knows about for node 'edge-node-1', then delete the route for acme/prod/chatbot/3 that goes via http://gateway:9090.", + "assertions": [ + { + "text": "Uses 'controller route list' (not 'node route list')", + "passed": null, + "evidence": null + }, + { + "text": "Passes -n edge-node-1 (or --node-id) to the list command", + "passed": null, + "evidence": null + }, + { + "text": "Uses 'controller route del' for deletion", + "passed": null, + "evidence": null + }, + { + "text": "Deletion command includes -n edge-node-1", + "passed": null, + "evidence": null + }, + { + "text": "Deletion command includes route acme/prod/chatbot/3", + "passed": null, + "evidence": null + }, + { + "text": "Deletion command includes 'via http://gateway:9090'", + "passed": null, + "evidence": null + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-2/with_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-2/with_skill/grading.json new file mode 100644 index 000000000..6a18a13de --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-2/with_skill/grading.json @@ -0,0 +1,36 @@ +{ + "eval_id": 2, + "run": "with_skill", + "expectations": [ + { + "text": "Uses 'controller route list' (not 'node route list')", + "passed": true, + "evidence": "slimctl controller route list -n edge-node-1" + }, + { + "text": "Passes -n edge-node-1 (or --node-id) to the list command", + "passed": true, + "evidence": "slimctl controller route list -n edge-node-1" + }, + { + "text": "Uses 'controller route del' for deletion", + "passed": true, + "evidence": "slimctl controller route del -n edge-node-1 acme/prod/chatbot/3 via http://gateway:9090" + }, + { + "text": "Deletion command includes -n edge-node-1", + "passed": true, + "evidence": "slimctl controller route del -n edge-node-1 ..." + }, + { + "text": "Deletion command includes route acme/prod/chatbot/3", + "passed": true, + "evidence": "... acme/prod/chatbot/3 via http://gateway:9090" + }, + { + "text": "Deletion command includes 'via http://gateway:9090'", + "passed": true, + "evidence": "... via http://gateway:9090" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-2/with_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-2/with_skill/outputs/response.md new file mode 100644 index 000000000..74d7f42cf --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-2/with_skill/outputs/response.md @@ -0,0 +1,28 @@ +# Managing Routes for 'edge-node-1' via the Control Plane + +## 1. List all routes for node 'edge-node-1' + +```bash +slimctl controller route list -n edge-node-1 +``` + +This queries the SLIM Control Plane for all routes associated with the node `edge-node-1`. + +## 2. Delete the route for acme/prod/chatbot/3 via http://gateway:9090 + +```bash +slimctl controller route del -n edge-node-1 acme/prod/chatbot/3 via http://gateway:9090 +``` + +This removes the specific route entry for `acme/prod/chatbot/3` that is routed through `http://gateway:9090` on the `edge-node-1` node. + +--- + +**Notes:** + +- Both commands use the `controller` subcommand (aliases: `c`, `ctrl`) since you are working with the control plane, not a single node's local API. +- The `-n` / `--node-id` flag specifies which node's routes to manage. +- If the control plane is not running at the default address (`127.0.0.1:46357`), add the `--server` flag: + ```bash + slimctl --server controller route list -n edge-node-1 + ``` diff --git a/skills/slimctl-workspace/iteration-1/eval-2/with_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-2/with_skill/timing.json new file mode 100644 index 000000000..032ad5ed3 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-2/with_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 14689, + "duration_ms": 21411, + "total_duration_seconds": 21.4 +} diff --git a/skills/slimctl-workspace/iteration-1/eval-2/without_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-2/without_skill/grading.json new file mode 100644 index 000000000..58aed32db --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-2/without_skill/grading.json @@ -0,0 +1,36 @@ +{ + "eval_id": 2, + "run": "without_skill", + "expectations": [ + { + "text": "Uses 'controller route list' (not 'node route list')", + "passed": true, + "evidence": "slimctl controller route list -n edge-node-1" + }, + { + "text": "Passes -n edge-node-1 (or --node-id) to the list command", + "passed": true, + "evidence": "slimctl controller route list -n edge-node-1" + }, + { + "text": "Uses 'controller route del' for deletion", + "passed": true, + "evidence": "slimctl controller route del -n edge-node-1 acme/prod/chatbot/3 via http://gateway:9090" + }, + { + "text": "Deletion command includes -n edge-node-1", + "passed": true, + "evidence": "slimctl controller route del -n edge-node-1 ..." + }, + { + "text": "Deletion command includes route acme/prod/chatbot/3", + "passed": true, + "evidence": "... acme/prod/chatbot/3 via http://gateway:9090" + }, + { + "text": "Deletion command includes 'via http://gateway:9090'", + "passed": true, + "evidence": "... via http://gateway:9090" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-2/without_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-2/without_skill/outputs/response.md new file mode 100644 index 000000000..baee90efb --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-2/without_skill/outputs/response.md @@ -0,0 +1,39 @@ +## Listing and Deleting Routes via the SLIM Control Plane + +To accomplish this task you need two `slimctl controller route` commands targeting the node `edge-node-1`. + +### Step 1: List all routes for node 'edge-node-1' + +Use the `controller route list` command with the `-n` flag to specify the node: + +```bash +slimctl controller route list -n edge-node-1 +``` + +This queries the SLIM Control Plane (not the node directly) and returns all routes it knows about for the node identified as `edge-node-1`. + +### Step 2: Delete the route for acme/prod/chatbot/3 via http://gateway:9090 + +Once you have confirmed the route exists in the listing, delete it with: + +```bash +slimctl controller route del -n edge-node-1 acme/prod/chatbot/3 via http://gateway:9090 +``` + +### Explanation + +- `controller route` commands talk to the SLIM Control Plane, which manages routes across multiple nodes. +- The `-n edge-node-1` (or `--node-id edge-node-1`) flag tells the control plane which node's routing table to operate on. +- The route `acme/prod/chatbot/3` follows the standard `org/namespace/agentname/agentid` format. +- The `via http://gateway:9090` portion identifies the specific next-hop connection to remove for that route. + +### Notes + +- If your control plane is not running on the default address (`127.0.0.1:46357`), use the `--server` global flag to point at it: + +```bash +slimctl --server controller route list -n edge-node-1 +slimctl --server controller route del -n edge-node-1 acme/prod/chatbot/3 via http://gateway:9090 +``` + +- You can verify the deletion was successful by re-running the list command afterward. diff --git a/skills/slimctl-workspace/iteration-1/eval-2/without_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-2/without_skill/timing.json new file mode 100644 index 000000000..c58ef6dab --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-2/without_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 17524, + "duration_ms": 38461, + "total_duration_seconds": 38.5 +} diff --git a/skills/slimctl-workspace/iteration-1/eval-3/eval_metadata.json b/skills/slimctl-workspace/iteration-1/eval-3/eval_metadata.json new file mode 100644 index 000000000..91d9dbf95 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-3/eval_metadata.json @@ -0,0 +1,32 @@ +{ + "eval_id": 3, + "eval_name": "configure-tls", + "prompt": "Set up slimctl to always connect to my SLIM node at myhost.internal:9090 using TLS with my CA cert at /etc/pki/ca.pem, client cert at /etc/pki/client.pem, and key at /etc/pki/client.key.", + "assertions": [ + { + "text": "Sets server to myhost.internal:9090 via 'config set server'", + "passed": null, + "evidence": null + }, + { + "text": "Sets tls-ca-file to /etc/pki/ca.pem", + "passed": null, + "evidence": null + }, + { + "text": "Sets tls-cert-file to /etc/pki/client.pem", + "passed": null, + "evidence": null + }, + { + "text": "Sets tls-key-file to /etc/pki/client.key", + "passed": null, + "evidence": null + }, + { + "text": "Disables tls-insecure (sets to false) since real TLS is being configured", + "passed": null, + "evidence": null + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-3/with_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-3/with_skill/grading.json new file mode 100644 index 000000000..baac97a5b --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-3/with_skill/grading.json @@ -0,0 +1,31 @@ +{ + "eval_id": 3, + "run": "with_skill", + "expectations": [ + { + "text": "Sets server to myhost.internal:9090 via 'config set server'", + "passed": true, + "evidence": "slimctl config set server myhost.internal:9090" + }, + { + "text": "Sets tls-ca-file to /etc/pki/ca.pem", + "passed": true, + "evidence": "slimctl config set tls-ca-file /etc/pki/ca.pem" + }, + { + "text": "Sets tls-cert-file to /etc/pki/client.pem", + "passed": true, + "evidence": "slimctl config set tls-cert-file /etc/pki/client.pem" + }, + { + "text": "Sets tls-key-file to /etc/pki/client.key", + "passed": true, + "evidence": "slimctl config set tls-key-file /etc/pki/client.key" + }, + { + "text": "Disables tls-insecure (sets to false) since real TLS is being configured", + "passed": false, + "evidence": "Response says 'no need to set tls-insecure to false' which is incorrect — the default is true (insecure) and must be explicitly disabled when using TLS certs" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-3/with_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-3/with_skill/outputs/response.md new file mode 100644 index 000000000..8571acbd0 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-3/with_skill/outputs/response.md @@ -0,0 +1,45 @@ +# Configuring slimctl for Persistent TLS Connection + +To make slimctl always connect to your SLIM node at `myhost.internal:9090` with mTLS using your specified certificates, run the following `slimctl config set` commands. These persist the settings to `~/.slimctl/config.yaml` so they apply to every future invocation without needing to pass flags each time. + +```bash +slimctl config set server myhost.internal:9090 +slimctl config set tls-ca-file /etc/pki/ca.pem +slimctl config set tls-cert-file /etc/pki/client.pem +slimctl config set tls-key-file /etc/pki/client.key +``` + +## What each command does + +| Command | Effect | +|---------|--------| +| `slimctl config set server myhost.internal:9090` | Sets the default gRPC endpoint to your SLIM node | +| `slimctl config set tls-ca-file /etc/pki/ca.pem` | Tells slimctl to use your CA certificate for verifying the server | +| `slimctl config set tls-cert-file /etc/pki/client.pem` | Sets the client certificate for mutual TLS authentication | +| `slimctl config set tls-key-file /etc/pki/client.key` | Sets the client private key for mutual TLS authentication | + +## Verifying the configuration + +After running the commands above, confirm your settings with: + +```bash +slimctl config list +``` + +This will display all persisted values from `~/.slimctl/config.yaml`. + +## Testing the connection + +Try listing routes to verify connectivity: + +```bash +slimctl node route list +``` + +If the connection succeeds, slimctl is correctly configured to use TLS with your CA and client certificates against `myhost.internal:9090`. + +## Notes + +- These settings are stored in `~/.slimctl/config.yaml` and apply to all commands by default. +- You can still override any setting for a single invocation using CLI flags (e.g., `--server`, `--tls.ca_file`, `--tls.cert_file`, `--tls.key_file`) or environment variables (e.g., `$SLIMCTL_COMMON_OPTS_SERVER`). +- Since you are providing a CA file, client cert, and client key, TLS is implicitly enabled (no need to set `tls-insecure` to false -- that is only the default for plain HTTP/2 connections). diff --git a/skills/slimctl-workspace/iteration-1/eval-3/with_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-3/with_skill/timing.json new file mode 100644 index 000000000..9fd242729 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-3/with_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 14930, + "duration_ms": 24521, + "total_duration_seconds": 24.5 +} diff --git a/skills/slimctl-workspace/iteration-1/eval-3/without_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-3/without_skill/grading.json new file mode 100644 index 000000000..5ba1f6cce --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-3/without_skill/grading.json @@ -0,0 +1,31 @@ +{ + "eval_id": 3, + "run": "without_skill", + "expectations": [ + { + "text": "Sets server to myhost.internal:9090 via 'config set server'", + "passed": true, + "evidence": "slimctl config set server myhost.internal:9090" + }, + { + "text": "Sets tls-ca-file to /etc/pki/ca.pem", + "passed": true, + "evidence": "slimctl config set tls-ca-file /etc/pki/ca.pem" + }, + { + "text": "Sets tls-cert-file to /etc/pki/client.pem", + "passed": true, + "evidence": "slimctl config set tls-cert-file /etc/pki/client.pem" + }, + { + "text": "Sets tls-key-file to /etc/pki/client.key", + "passed": true, + "evidence": "slimctl config set tls-key-file /etc/pki/client.key" + }, + { + "text": "Disables tls-insecure (sets to false) since real TLS is being configured", + "passed": true, + "evidence": "slimctl config set tls-insecure false — explicitly included as step 2" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-3/without_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-3/without_skill/outputs/response.md new file mode 100644 index 000000000..d8dea9f8a --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-3/without_skill/outputs/response.md @@ -0,0 +1,39 @@ +To configure slimctl to always connect to your SLIM node at `myhost.internal:9090` using mTLS with your specified certificates, you need to persist these settings in the slimctl configuration file. Run the following `slimctl config set` commands: + +```bash +# Set the server address +slimctl config set server myhost.internal:9090 + +# Disable insecure mode (plain HTTP/2) since you are using real TLS +slimctl config set tls-insecure false + +# Set the CA certificate for verifying the server +slimctl config set tls-ca-file /etc/pki/ca.pem + +# Set the client certificate for mutual TLS +slimctl config set tls-cert-file /etc/pki/client.pem + +# Set the client private key +slimctl config set tls-key-file /etc/pki/client.key +``` + +These settings are persisted to `~/.slimctl/config.yaml` and will be used for every subsequent slimctl command. The resulting config file will look like this: + +```yaml +common_opts: + server: "myhost.internal:9090" + timeout: "15s" + tls_insecure: false + tls_insecure_skip_verify: false + tls_ca_file: "/etc/pki/ca.pem" + tls_cert_file: "/etc/pki/client.pem" + tls_key_file: "/etc/pki/client.key" +``` + +After running these commands, all slimctl operations (e.g., `slimctl node route list`, `slimctl controller node list`) will automatically connect to `myhost.internal:9090` over TLS using your CA and client certificates without needing to pass flags each time. + +You can verify the configuration at any time with: + +```bash +slimctl config list +``` diff --git a/skills/slimctl-workspace/iteration-1/eval-3/without_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-3/without_skill/timing.json new file mode 100644 index 000000000..16096b260 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-3/without_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 24353, + "duration_ms": 41292, + "total_duration_seconds": 41.3 +} diff --git a/skills/slimctl-workspace/iteration-1/eval-4/eval_metadata.json b/skills/slimctl-workspace/iteration-1/eval-4/eval_metadata.json new file mode 100644 index 000000000..a484acd32 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-4/eval_metadata.json @@ -0,0 +1,32 @@ +{ + "eval_id": 4, + "eval_name": "channel-and-participants", + "prompt": "I want to create a SLIM channel with alice as a moderator, then add participants bob and carol to it, and finally list the participants. Walk me through the slimctl commands.", + "assertions": [ + { + "text": "Uses 'controller channel create' with moderators=alice", + "passed": null, + "evidence": null + }, + { + "text": "Uses 'controller participant add bob -c '", + "passed": null, + "evidence": null + }, + { + "text": "Uses 'controller participant add carol -c '", + "passed": null, + "evidence": null + }, + { + "text": "Uses 'controller participant list -c '", + "passed": null, + "evidence": null + }, + { + "text": "Explains that the channel ID comes from the output of the channel create step", + "passed": null, + "evidence": null + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-4/with_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-4/with_skill/grading.json new file mode 100644 index 000000000..f87c073ea --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-4/with_skill/grading.json @@ -0,0 +1,31 @@ +{ + "eval_id": 4, + "run": "with_skill", + "expectations": [ + { + "text": "Uses 'controller channel create' with moderators=alice", + "passed": true, + "evidence": "slimctl controller channel create moderators=alice" + }, + { + "text": "Uses 'controller participant add bob -c '", + "passed": true, + "evidence": "slimctl controller participant add bob -c " + }, + { + "text": "Uses 'controller participant add carol -c '", + "passed": true, + "evidence": "slimctl controller participant add carol -c " + }, + { + "text": "Uses 'controller participant list -c '", + "passed": true, + "evidence": "slimctl controller participant list -c " + }, + { + "text": "Explains that the channel ID comes from the output of the channel create step", + "passed": true, + "evidence": "The placeholder should be replaced with the actual ID returned by the 'channel create' command" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-4/with_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-4/with_skill/outputs/response.md new file mode 100644 index 000000000..82c4c5e28 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-4/with_skill/outputs/response.md @@ -0,0 +1,45 @@ +# Creating a SLIM Channel, Adding Participants, and Listing Them + +Here is a walkthrough of the `slimctl` commands to create a channel with `alice` as a moderator, add `bob` and `carol` as participants, and then list all participants. + +## Step 1: Create the Channel with Alice as Moderator + +```bash +slimctl controller channel create moderators=alice +``` + +This command talks to the SLIM Control Plane and creates a new MLS-backed channel with `alice` designated as a moderator. The command will return a **channel ID** (e.g., something like `ch-abc123`) that you will need for the subsequent steps. + +## Step 2: Add Participants Bob and Carol + +Using the channel ID returned from step 1, add `bob` and `carol` as participants: + +```bash +slimctl controller participant add bob -c +slimctl controller participant add carol -c +``` + +Replace `` with the actual ID returned when you created the channel. + +## Step 3: List Participants + +To verify that both participants were added successfully, list all participants on the channel: + +```bash +slimctl controller participant list -c +``` + +This will display all participants currently in the channel, which should include `bob` and `carol` (and potentially `alice` as the moderator, depending on how the control plane reports membership). + +## Notes + +- All `controller` commands talk to the SLIM Control Plane (not a single node). Make sure your slimctl is pointed at the correct control plane server. The default is `127.0.0.1:46357`. +- If your control plane is running elsewhere, use the `--server` flag or configure it persistently: + ```bash + slimctl config set server + ``` +- The `controller` subcommand can also be shortened to `c` or `ctrl`: + ```bash + slimctl ctrl channel create moderators=alice + slimctl ctrl participant add bob -c + ``` diff --git a/skills/slimctl-workspace/iteration-1/eval-4/with_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-4/with_skill/timing.json new file mode 100644 index 000000000..ac00929dc --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-4/with_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 14782, + "duration_ms": 24502, + "total_duration_seconds": 24.5 +} diff --git a/skills/slimctl-workspace/iteration-1/eval-4/without_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-4/without_skill/grading.json new file mode 100644 index 000000000..926082ec2 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-4/without_skill/grading.json @@ -0,0 +1,31 @@ +{ + "eval_id": 4, + "run": "without_skill", + "expectations": [ + { + "text": "Uses 'controller channel create' with moderators=alice", + "passed": true, + "evidence": "slimctl controller channel create moderators=alice" + }, + { + "text": "Uses 'controller participant add bob -c '", + "passed": true, + "evidence": "slimctl controller participant add bob -c " + }, + { + "text": "Uses 'controller participant add carol -c '", + "passed": true, + "evidence": "slimctl controller participant add carol -c " + }, + { + "text": "Uses 'controller participant list -c '", + "passed": true, + "evidence": "slimctl controller participant list -c " + }, + { + "text": "Explains that the channel ID comes from the output of the channel create step", + "passed": true, + "evidence": "Agent read source code and correctly identified this workflow" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-4/without_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-4/without_skill/outputs/response.md new file mode 100644 index 000000000..7e6ae38f6 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-4/without_skill/outputs/response.md @@ -0,0 +1,93 @@ +# Creating a SLIM Channel and Managing Participants with slimctl + +Here is a walkthrough of the `slimctl` commands to create a channel with `alice` as a moderator, add `bob` and `carol` as participants, and then list the participants. + +## Prerequisites + +Before running these commands, ensure that: + +1. `slimctl` is built and available in your PATH. +2. A SLIM control plane is running and accessible. +3. Your `slimctl` configuration is set to point to the correct control plane server (you can verify with `slimctl config list` or set it with `slimctl config set server `). + +## Step 1: Create the Channel with Alice as Moderator + +```bash +slimctl controller channel create moderators=alice +``` + +This command creates a new channel (MLS group) with `alice` designated as a moderator. The argument uses the format `moderators=`. If the command succeeds, it will output something like: + +``` +Channel created successfully: +``` + +Take note of the returned `` -- you will need it in subsequent commands. + +If you wanted multiple moderators at creation time, you could do: + +```bash +slimctl controller channel create moderators=alice,dave +``` + +## Step 2: Add Participants Bob and Carol + +Once you have the channel name from step 1, add `bob` and `carol` as participants: + +```bash +slimctl controller participant add bob -c +slimctl controller participant add carol -c +``` + +The `-c` flag (short for `--channel-id`) specifies which channel to add the participant to. Each successful addition will produce output like: + +``` +Adding participant to channel : bob +Participant added successfully to channel : bob +``` + +## Step 3: List Participants + +To verify that both participants were added correctly, list all participants in the channel: + +```bash +slimctl controller participant list -c +``` + +This will output the list of participants registered in the channel: + +``` +Listing participants for channel ID: +Following participants found for channel : ["bob", "carol"] +``` + +## Complete Example + +Putting it all together (assuming the channel name returned is `my-channel`): + +```bash +# Create channel with alice as moderator +slimctl controller channel create moderators=alice +# Output: Channel created successfully: my-channel + +# Add bob as a participant +slimctl controller participant add bob -c my-channel + +# Add carol as a participant +slimctl controller participant add carol -c my-channel + +# List all participants +slimctl controller participant list -c my-channel +``` + +## Additional Useful Commands + +- **List all channels:** `slimctl controller channel list` +- **Delete a participant:** `slimctl controller participant delete bob -c my-channel` +- **Delete a channel:** `slimctl controller channel delete my-channel` + +## Notes + +- The `controller` subcommand can also be invoked using its aliases: `c` or `ctrl`. For example: `slimctl c channel create moderators=alice`. +- The `participant list` subcommand also supports the `ls` alias: `slimctl controller participant ls -c my-channel`. +- If TLS is not configured, you may need to pass `--tls.insecure` to disable TLS for the gRPC connection to the control plane. diff --git a/skills/slimctl-workspace/iteration-1/eval-4/without_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-4/without_skill/timing.json new file mode 100644 index 000000000..be84e9483 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-4/without_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 41229, + "duration_ms": 60806, + "total_duration_seconds": 60.8 +} diff --git a/skills/slimctl-workspace/iteration-1/eval-5/eval_metadata.json b/skills/slimctl-workspace/iteration-1/eval-5/eval_metadata.json new file mode 100644 index 000000000..fb6d9a302 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-5/eval_metadata.json @@ -0,0 +1,22 @@ +{ + "eval_id": 5, + "eval_name": "list-routes-and-connections", + "prompt": "Show me all the routes and connections currently active on my local SLIM node.", + "assertions": [ + { + "text": "Uses 'node route list' (direct node command, not controller)", + "passed": null, + "evidence": null + }, + { + "text": "Uses 'node connection list' (or alias 'node conn list')", + "passed": null, + "evidence": null + }, + { + "text": "Mentions default server 127.0.0.1:46357 or explains the local node assumption", + "passed": null, + "evidence": null + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-5/with_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-5/with_skill/grading.json new file mode 100644 index 000000000..dd4f9b763 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-5/with_skill/grading.json @@ -0,0 +1,21 @@ +{ + "eval_id": 5, + "run": "with_skill", + "expectations": [ + { + "text": "Uses 'node route list' (direct node command, not controller)", + "passed": true, + "evidence": "slimctl node route list" + }, + { + "text": "Uses 'node connection list' (or alias 'node conn list')", + "passed": true, + "evidence": "slimctl node connection list" + }, + { + "text": "Mentions default server 127.0.0.1:46357 or explains the local node assumption", + "passed": true, + "evidence": "Both commands default to connecting to 127.0.0.1:46357 (the local SLIM node control API)" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-5/with_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-5/with_skill/outputs/response.md new file mode 100644 index 000000000..710be60a9 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-5/with_skill/outputs/response.md @@ -0,0 +1,29 @@ +# Viewing Active Routes and Connections on Your Local SLIM Node + +To show all routes and connections currently active on your local SLIM node, run these two commands: + +## List all routes + +```bash +slimctl node route list +``` + +## List all connections + +```bash +slimctl node connection list +``` + +By default, these commands connect to `127.0.0.1:46357` (the default local SLIM node control API endpoint). If your node is running on a different address or port, specify it with the `--server` flag: + +```bash +slimctl --server node route list +slimctl --server node connection list +``` + +## Notes + +- The `node` subcommand (aliases: `n`, `instance`, `i`) talks directly to a single SLIM node's local control API. +- Routes are displayed in the format `org/namespace/agentname/agentid` (e.g., `acme/production/assistant/1`). +- The `connection list` command (alias: `conn`) shows all active connections the node has to other SLIM nodes. +- If your node requires authentication, add `--basic-auth-creds user:pass` or the appropriate TLS flags (`--tls.ca_file`, `--tls.cert_file`, `--tls.key_file`). diff --git a/skills/slimctl-workspace/iteration-1/eval-5/with_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-5/with_skill/timing.json new file mode 100644 index 000000000..9a81df07d --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-5/with_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 14990, + "duration_ms": 24214, + "total_duration_seconds": 24.2 +} diff --git a/skills/slimctl-workspace/iteration-1/eval-5/without_skill/grading.json b/skills/slimctl-workspace/iteration-1/eval-5/without_skill/grading.json new file mode 100644 index 000000000..7268866fa --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-5/without_skill/grading.json @@ -0,0 +1,21 @@ +{ + "eval_id": 5, + "run": "without_skill", + "expectations": [ + { + "text": "Uses 'node route list' (direct node command, not controller)", + "passed": true, + "evidence": "slimctl node route list" + }, + { + "text": "Uses 'node connection list' (or alias 'node conn list')", + "passed": true, + "evidence": "slimctl node connection list — agent read source code" + }, + { + "text": "Mentions default server 127.0.0.1:46357 or explains the local node assumption", + "passed": true, + "evidence": "Both commands communicate with the SLIM node's gRPC control API via the OpenControlChannel streaming RPC — agent derived this from source" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-1/eval-5/without_skill/outputs/response.md b/skills/slimctl-workspace/iteration-1/eval-5/without_skill/outputs/response.md new file mode 100644 index 000000000..6637dfb7c --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-5/without_skill/outputs/response.md @@ -0,0 +1,78 @@ +# Viewing Routes and Connections on Your Local SLIM Node + +To show all routes and connections currently active on your local SLIM node, you need to use the `slimctl` CLI tool with the `node` subcommand. There are two separate commands -- one for routes and one for connections. + +## List All Active Routes + +```bash +slimctl node route list +``` + +This connects to your local SLIM node's gRPC control API and retrieves the current subscription/routing table. The output shows each route entry in the format: + +``` +// id= local=[...] remote=[...] +``` + +Each entry includes: +- The hierarchical name components (organization, namespace, agent type) +- The agent ID +- Local connections associated with that route (with connection ID, config data, link ID, and direction) +- Remote connections associated with that route + +You can also use the alias `ls` instead of `list`: + +```bash +slimctl node route ls +``` + +## List All Active Connections + +```bash +slimctl node connection list +``` + +This retrieves all active connections on the node. The output shows each connection in the format: + +``` +id= direction= link_id= +``` + +You can also use the alias `conn` for `connection`: + +```bash +slimctl node conn list +``` + +## Configuration + +By default, `slimctl` connects to `localhost:46357` (the default SLIM gRPC control API endpoint). If your node is running on a different address, you can specify it with: + +```bash +slimctl --server node route list +slimctl --server node connection list +``` + +Or set it persistently: + +```bash +slimctl config set server +``` + +If your node is running without TLS (common in local development), add the `--tls.insecure` flag: + +```bash +slimctl --tls.insecure node route list +slimctl --tls.insecure node connection list +``` + +## Combined View + +To see both routes and connections together, simply run both commands: + +```bash +slimctl node route list +slimctl node connection list +``` + +There is no single command that combines both outputs into one view -- they must be queried separately. diff --git a/skills/slimctl-workspace/iteration-1/eval-5/without_skill/timing.json b/skills/slimctl-workspace/iteration-1/eval-5/without_skill/timing.json new file mode 100644 index 000000000..dccc51424 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/eval-5/without_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 38167, + "duration_ms": 41209, + "total_duration_seconds": 41.2 +} diff --git a/skills/slimctl-workspace/iteration-1/feedback.json b/skills/slimctl-workspace/iteration-1/feedback.json new file mode 100644 index 000000000..a9e35b757 --- /dev/null +++ b/skills/slimctl-workspace/iteration-1/feedback.json @@ -0,0 +1,15 @@ +{ + "reviews": [ + { + "run_id": "eval-1-with_skill", + "feedback": "Looks good ", + "timestamp": "2026-05-18T13:04:18.167Z" + }, + { + "run_id": "eval-1-without_skill", + "feedback": "looks good", + "timestamp": "2026-05-18T13:04:18.167Z" + } + ], + "status": "in_progress" +} diff --git a/skills/slimctl-workspace/iteration-2/eval-3/eval_metadata.json b/skills/slimctl-workspace/iteration-2/eval-3/eval_metadata.json new file mode 100644 index 000000000..91d9dbf95 --- /dev/null +++ b/skills/slimctl-workspace/iteration-2/eval-3/eval_metadata.json @@ -0,0 +1,32 @@ +{ + "eval_id": 3, + "eval_name": "configure-tls", + "prompt": "Set up slimctl to always connect to my SLIM node at myhost.internal:9090 using TLS with my CA cert at /etc/pki/ca.pem, client cert at /etc/pki/client.pem, and key at /etc/pki/client.key.", + "assertions": [ + { + "text": "Sets server to myhost.internal:9090 via 'config set server'", + "passed": null, + "evidence": null + }, + { + "text": "Sets tls-ca-file to /etc/pki/ca.pem", + "passed": null, + "evidence": null + }, + { + "text": "Sets tls-cert-file to /etc/pki/client.pem", + "passed": null, + "evidence": null + }, + { + "text": "Sets tls-key-file to /etc/pki/client.key", + "passed": null, + "evidence": null + }, + { + "text": "Disables tls-insecure (sets to false) since real TLS is being configured", + "passed": null, + "evidence": null + } + ] +} diff --git a/skills/slimctl-workspace/iteration-2/eval-3/with_skill/grading.json b/skills/slimctl-workspace/iteration-2/eval-3/with_skill/grading.json new file mode 100644 index 000000000..d306e4a9e --- /dev/null +++ b/skills/slimctl-workspace/iteration-2/eval-3/with_skill/grading.json @@ -0,0 +1,31 @@ +{ + "eval_id": 3, + "run": "with_skill", + "expectations": [ + { + "text": "Sets server to myhost.internal:9090 via 'config set server'", + "passed": true, + "evidence": "slimctl config set server myhost.internal:9090" + }, + { + "text": "Sets tls-ca-file to /etc/pki/ca.pem", + "passed": true, + "evidence": "slimctl config set tls-ca-file /etc/pki/ca.pem" + }, + { + "text": "Sets tls-cert-file to /etc/pki/client.pem", + "passed": true, + "evidence": "slimctl config set tls-cert-file /etc/pki/client.pem" + }, + { + "text": "Sets tls-key-file to /etc/pki/client.key", + "passed": true, + "evidence": "slimctl config set tls-key-file /etc/pki/client.key" + }, + { + "text": "Disables tls-insecure (sets to false) since real TLS is being configured", + "passed": true, + "evidence": "slimctl config set tls-insecure false — listed as second command with explicit callout: 'Even if you provide cert and CA file paths, they will be ignored unless you explicitly set tls-insecure to false'" + } + ] +} diff --git a/skills/slimctl-workspace/iteration-2/eval-3/with_skill/outputs/response.md b/skills/slimctl-workspace/iteration-2/eval-3/with_skill/outputs/response.md new file mode 100644 index 000000000..d83dc0ff0 --- /dev/null +++ b/skills/slimctl-workspace/iteration-2/eval-3/with_skill/outputs/response.md @@ -0,0 +1,21 @@ +# Configuring slimctl for Persistent TLS Connection + +To permanently configure slimctl to connect to your SLIM node at `myhost.internal:9090` with mTLS using your certificates, run the following commands: + +```bash +slimctl config set server myhost.internal:9090 +slimctl config set tls-insecure false +slimctl config set tls-ca-file /etc/pki/ca.pem +slimctl config set tls-cert-file /etc/pki/client.pem +slimctl config set tls-key-file /etc/pki/client.key +``` + +**Important:** The `tls-insecure false` setting is critical. By default, slimctl sets `tls-insecure` to `true`, which means it uses plain HTTP/2 with no TLS. Even if you provide cert and CA file paths, they will be ignored unless you explicitly set `tls-insecure` to `false`. This command disables the insecure mode and enables TLS on the connection. + +These settings are persisted in `~/.slimctl/config.yaml`, so every subsequent `slimctl` invocation will automatically use this server address and TLS configuration without needing to pass flags each time. + +You can verify your configuration at any time with: + +```bash +slimctl config list +``` diff --git a/skills/slimctl-workspace/iteration-2/eval-3/with_skill/timing.json b/skills/slimctl-workspace/iteration-2/eval-3/with_skill/timing.json new file mode 100644 index 000000000..fe31b509a --- /dev/null +++ b/skills/slimctl-workspace/iteration-2/eval-3/with_skill/timing.json @@ -0,0 +1,5 @@ +{ + "total_tokens": 15000, + "duration_ms": 18769, + "total_duration_seconds": 18.8 +} diff --git a/skills/slimctl-workspace/trigger-evals.json b/skills/slimctl-workspace/trigger-evals.json new file mode 100644 index 000000000..d227063ff --- /dev/null +++ b/skills/slimctl-workspace/trigger-evals.json @@ -0,0 +1,82 @@ +[ + { + "query": "how do i add a route for my agent acme/prod/assistant/0 on my SLIM node? it needs to forward to http://10.0.1.5:8080", + "should_trigger": true + }, + { + "query": "slimctl keeps saying connection refused when i run controller route list -n node1, what am i doing wrong", + "should_trigger": true + }, + { + "query": "i want to create a SLIM channel with alice and bob as moderators and then add charlie as a participant", + "should_trigger": true + }, + { + "query": "what's the slimctl command to see all the connections on my local SLIM instance?", + "should_trigger": true + }, + { + "query": "need to set up slimctl to talk to a remote node at prod-slim.mycompany.com:9090 over mtls, i have the certs in /etc/pki/", + "should_trigger": true + }, + { + "query": "how do i start a slim node locally with slimctl using a custom config file at /etc/slim/prod.yaml", + "should_trigger": true + }, + { + "query": "list all nodes registered in the SLIM control plane", + "should_trigger": true + }, + { + "query": "i need to delete the route org/ns/bot/2 from edge-node-1 via the control plane, not directly on the node", + "should_trigger": true + }, + { + "query": "what does `slimctl config set tls-insecure false` actually do? why do i need it if i already set tls-ca-file?", + "should_trigger": true + }, + { + "query": "can i use environment variables instead of flags to configure the slimctl server address and credentials", + "should_trigger": true + }, + { + "query": "how do i add a static route in linux for the 192.168.1.0/24 subnet via gateway 10.0.0.1", + "should_trigger": false + }, + { + "query": "i'm setting up mutual TLS for my go grpc server, which tls.Config fields do i need to set for client certificate verification", + "should_trigger": false + }, + { + "query": "how do i create a channel in slack via the api and invite users to it programmatically", + "should_trigger": false + }, + { + "query": "my kubernetes ingress isn't routing traffic to the right service, here's my ingress.yaml - what's wrong with the path rules", + "should_trigger": false + }, + { + "query": "i'm using the agntcy python SDK to send a message from one agent to another, how do i set up the session", + "should_trigger": false + }, + { + "query": "how do i configure a rabbitmq exchange and bind queues to route messages by routing key", + "should_trigger": false + }, + { + "query": "need to list all active tcp connections on my linux server and see which ports are listening", + "should_trigger": false + }, + { + "query": "how do i manage participants in a google meet or zoom call via their REST api", + "should_trigger": false + }, + { + "query": "setting up envoy proxy to route gRPC traffic between microservices with TLS termination", + "should_trigger": false + }, + { + "query": "i want to check what routes my istio service mesh has configured for my payment-service", + "should_trigger": false + } +] \ No newline at end of file diff --git a/skills/slimctl/SKILL.md b/skills/slimctl/SKILL.md new file mode 100644 index 000000000..c03d2f229 --- /dev/null +++ b/skills/slimctl/SKILL.md @@ -0,0 +1,214 @@ +--- +name: slimctl +description: > + Use this skill whenever the user mentions slimctl or SLIM nodes. slimctl is a + proprietary CLI — not a standard system tool — for managing SLIM (Secure + Low-Latency Interactive Messaging) infrastructure. Always invoke for: starting + or configuring a local SLIM node, connecting slimctl to a remote server, + setting up TLS or mTLS for slimctl, configuring slimctl via flags/env + vars/config files, adding or listing routes on a SLIM node, querying the SLIM + control plane (nodes, routes, channels, participants), or troubleshooting + slimctl connection errors. The primary triggers are the words "slimctl" or + "SLIM node/control plane" anywhere in the query. +--- + +# slimctl — SLIM Control CLI + +> **Note:** slimctl is a proprietary tool with its own specific command syntax. +> Do not guess or invent commands from general knowledge — use only the +> reference below. + +slimctl manages SLIM nodes and the SLIM Control Plane over gRPC. There are two +target audiences for commands: + +- **`node`** — talks directly to a single SLIM node's local control API +- **`controller`** — talks to the SLIM Control Plane, which manages multiple nodes + +## Defaults + +| Setting | Default | +|---|---| +| Server | `127.0.0.1:46357` | +| Timeout | `15s` | +| TLS | **insecure = true** (plain HTTP/2, no TLS) | +| Config file | `~/.slimctl/config.yaml` | + +> **Important:** `tls-insecure` is `true` by default. Providing cert/CA files alone does NOT +> enable TLS — you must also explicitly set `tls-insecure false`, otherwise the connection +> stays plain HTTP/2 and the cert files are ignored. + +## Global flags (apply to every command) + +``` +--config / $SLIMCTL_CONFIG path to config file +-s, --server / $SLIMCTL_COMMON_OPTS_SERVER gRPC endpoint host:port +-b, --basic-auth-creds / $SLIMCTL_COMMON_OPTS_BASIC_AUTH_CREDS user:pass + --timeout / $SLIMCTL_COMMON_OPTS_TIMEOUT duration e.g. 15s, 1m + --tls.insecure disable TLS (plain HTTP/2) + --tls.insecure_skip_verify skip cert verification + --tls.ca_file path to CA cert + --tls.cert_file path to client cert + --tls.key_file path to client key +``` + +--- + +## Route format + +Routes are written as four slash-separated components: + +``` +org/namespace/agentname/agentid +``` + +Example: `acme/production/assistant/1` + +--- + +## Commands + +### version + +```bash +slimctl version +``` + +### config + +Persistent settings live in `~/.slimctl/config.yaml`. CLI flags and env vars +always override the file for a single invocation. + +```bash +slimctl config list +slimctl config set server +slimctl config set timeout # e.g. 30s, 2m +slimctl config set tls-insecure +slimctl config set tls-insecure-skip-verify +slimctl config set tls-ca-file +slimctl config set tls-cert-file +slimctl config set tls-key-file +slimctl config set basic-auth-creds +``` + +### node (aliases: `n`, `instance`, `i`) + +Talks directly to one SLIM node. Use the global `--server` flag to point at it. + +```bash +# Routes +slimctl node route list +slimctl node route add via +slimctl node route del via + +# Connections +slimctl node connection list # alias: conn +``` + +The connection config JSON for `route add` needs at minimum: +```json +{ "endpoint": "http://host:port" } +``` + +### controller (aliases: `c`, `ctrl`) + +Talks to the SLIM Control Plane. + +```bash +# Nodes +slimctl controller node list + +# Connections (require -n / --node-id) +slimctl controller connection list -n + +# Routes (require -n / --node-id) +slimctl controller route list -n +slimctl controller route add -n via +slimctl controller route del -n via +slimctl controller route outline [-o ] [-t ] + +# Links +slimctl controller link list + +# Channels (MLS groups) +slimctl controller channel create +slimctl controller channel delete +slimctl controller channel list + +# Participants +slimctl controller participant add -c +slimctl controller participant delete -c +slimctl controller participant list -c +``` + +### slim (alias: `s`) + +Starts a local SLIM node in the foreground. `--config` and `--endpoint` are mutually exclusive. + +```bash +slimctl slim start # default endpoint 127.0.0.1:46357 +slimctl slim start --endpoint 0.0.0.0:9090 +slimctl slim start -c /etc/slim/config.yaml +``` + +--- + +## Common workflows + +### Point slimctl at a different node + +```bash +# One-off +slimctl --server myhost:9090 node route list + +# Persist +slimctl config set server myhost:9090 +``` + +### mTLS connection + +Remember: `tls-insecure` defaults to `true`, so you must turn it off or TLS will not be used +even when cert files are provided. + +**Persist mTLS settings** (recommended): +```bash +slimctl config set tls-insecure false # REQUIRED — disables plain HTTP/2 mode +slimctl config set tls-ca-file /etc/ssl/ca.pem +slimctl config set tls-cert-file /etc/ssl/client.pem +slimctl config set tls-key_file /etc/ssl/client.key +slimctl config set server myhost:9090 +``` + +**One-off mTLS flag**: +```bash +slimctl --tls.insecure=false \ + --tls.ca_file /etc/ssl/ca.pem \ + --tls.cert_file /etc/ssl/client.pem \ + --tls.key_file /etc/ssl/client.key \ + node route list +``` + +### Add a route directly on a node + +```bash +# 1. Create a connection config file +cat > /tmp/conn.json <<'EOF' +{ "endpoint": "http://remote-node:8080" } +EOF + +# 2. Add the route +slimctl node route add acme/prod/assistant/1 via /tmp/conn.json +``` + +### Manage a channel through the control plane + +```bash +# Create channel with a moderator +slimctl controller channel create moderators=alice + +# Add participants +slimctl controller participant add bob -c +slimctl controller participant add carol -c + +# List participants +slimctl controller participant list -c +``` diff --git a/skills/slimctl/evals/evals.json b/skills/slimctl/evals/evals.json new file mode 100644 index 000000000..a78e9f4a0 --- /dev/null +++ b/skills/slimctl/evals/evals.json @@ -0,0 +1,35 @@ +{ + "skill_name": "slimctl", + "evals": [ + { + "id": 1, + "prompt": "I need to add a route on my SLIM node for agent acme/production/assistant/1 that goes via the remote node at http://10.0.0.5:8080. How do I do this with slimctl?", + "expected_output": "A correct slimctl command using 'node route add' with the route in org/ns/agent/id format, a connection config JSON file, and the 'via' keyword.", + "files": [] + }, + { + "id": 2, + "prompt": "List all routes the control plane knows about for node 'edge-node-1', then delete the route for acme/prod/chatbot/3 that goes via http://gateway:9090.", + "expected_output": "Correct slimctl controller commands: 'controller route list -n edge-node-1' and 'controller route del -n edge-node-1 acme/prod/chatbot/3 via http://gateway:9090'.", + "files": [] + }, + { + "id": 3, + "prompt": "Set up slimctl to always connect to my SLIM node at myhost.internal:9090 using TLS with my CA cert at /etc/pki/ca.pem, client cert at /etc/pki/client.pem, and key at /etc/pki/client.key.", + "expected_output": "Correct slimctl config set commands for server, tls-ca-file, tls-cert-file, tls-key-file, and turning off tls-insecure.", + "files": [] + }, + { + "id": 4, + "prompt": "I want to create a SLIM channel with alice as a moderator, then add participants bob and carol to it, and finally list the participants. Walk me through the slimctl commands.", + "expected_output": "Correct sequence: controller channel create moderators=alice, then controller participant add bob/carol -c , then controller participant list -c .", + "files": [] + }, + { + "id": 5, + "prompt": "Show me all the routes and connections currently active on my local SLIM node.", + "expected_output": "Correct slimctl commands: 'node route list' and 'node connection list' (both talk directly to the node at default 127.0.0.1:46357).", + "files": [] + } + ] +} From e9dfa66613083aa76a1a762d6bd2b2381cde6424 Mon Sep 17 00:00:00 2001 From: Sam Betts <1769706+Tehsmash@users.noreply.github.com> Date: Tue, 19 May 2026 11:21:34 +0100 Subject: [PATCH 2/2] fix(ci): avoid shell injection in publish-skill job Pass github.ref_name through an env var rather than interpolating it directly into the shell command, as flagged by zizmor. Signed-off-by: Sam Betts <1769706+Tehsmash@users.noreply.github.com> --- .github/workflows/release-slimctl.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-slimctl.yaml b/.github/workflows/release-slimctl.yaml index 3b68d54ec..8166b4ea7 100644 --- a/.github/workflows/release-slimctl.yaml +++ b/.github/workflows/release-slimctl.yaml @@ -132,8 +132,9 @@ jobs: - name: Upload skill to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} run: | - gh release upload "${{ github.ref_name }}" slimctl.skill --clobber + gh release upload "$TAG" slimctl.skill --clobber homebrew: runs-on: ubuntu-latest