feat(ui): add KG manage workspace, data sources maintenance, and extraction chat

Makefile

@@ -23,8 +23,9 @@ certs:
 .PHONY: dev
 dev: certs
 	@echo "🧰 [Development] Starting application containers..."
+	docker compose -f compose.yaml -f compose.dev.yaml --profile build-only build agent-runtime
 	docker compose -f compose.yaml build
-	docker compose -f compose.yaml -f compose.dev.yaml --profile ui up -d
+	HOST_UID=$$(id -u) HOST_GID=$$(id -g) docker compose -f compose.yaml -f compose.dev.yaml --profile ui up -d
 	@echo "Done."
 	@echo "----------------------------"
 	@echo "API Root: http://localhost:8000"
@@ -35,6 +36,9 @@ dev: certs
 .PHONY: down
 down:
 	docker compose -f compose.yaml -f compose.dev.yaml down
+	@echo "Stopping Graph Management sticky and worker containers..."
+	-@docker ps -aq --filter name=kartograph-sticky- | xargs -r docker rm -f
+	-@docker ps -aq --filter name=kartograph-worker- | xargs -r docker rm -f
 
 
 .PHONY: run

compose.dev.yaml

@@ -1,14 +1,43 @@
 # Development overrides for compose.yaml
 services:
+  agent-runtime:
+    build:
+      context: ./src/agent-runtime
+      dockerfile: Dockerfile
+    image: kartograph-agent-runtime:dev
+    profiles: ["build-only"]
+
   api:
-    # Run as root in dev to handle host file permissions (any umask)
-    user: "${UID}:${GID}"
+    # Root required for Docker-out-of-Docker via mounted /var/run/docker.sock in dev
+    user: "0:0"
     environment:
       UV_CACHE_DIR: /tmp/uv-cache
+      HOST_UID: ${HOST_UID}
+      HOST_GID: ${HOST_GID}
+      KARTOGRAPH_EXTRACTION_RUNTIME_BACKEND: container
+      KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_ENGINE: auto
+      KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_NETWORK: kartograph_kartograph
+      KARTOGRAPH_EXTRACTION_RUNTIME_STICKY_IMAGE: kartograph-agent-runtime:dev
+      KARTOGRAPH_EXTRACTION_RUNTIME_API_BASE_URL: http://api:8000
+      KARTOGRAPH_EXTRACTION_RUNTIME_JOB_PACKAGE_WORK_DIR: /tmp/kartograph/job_packages
+      KARTOGRAPH_EXTRACTION_RUNTIME_SKILLS_DIR: ${PWD}/skills
+      KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_RUN_UID: ${HOST_UID}
+      KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_RUN_GID: ${HOST_GID}
+      # Vertex AI for Claude Agent SDK in sticky assistant containers
+      CLAUDE_CODE_USE_VERTEX: "1"
+      ANTHROPIC_VERTEX_PROJECT_ID: itpc-gcp-hcm-pe-eng-claude
+      CLOUD_ML_REGION: us-east5
+      KARTOGRAPH_GCLOUD_CONFIG_MOUNT: ${HOME}/.config/gcloud
     volumes:
       # Mount the entire app directory  (minus  venv) for hot-reload
       - ./src/api:/app:z
       - /app/.venv
+      # Shared with sibling sticky containers launched via the host Docker socket
+      - /tmp/kartograph/job_packages:/tmp/kartograph/job_packages
+      # Allow API process to launch sibling extraction runtime containers locally
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Docker/Podman CLI from host (required for container runtime backend)
+      - ${DOCKER_BIN:-/usr/bin/docker}:/usr/bin/docker:ro
     command:
       - /bin/bash
       - -c

compose.yaml

@@ -145,6 +145,7 @@ services:
       - kartograph
     volumes:
       - ./certs:/certs:ro
+      - ./skills:/app/skills:ro
       # Mount host CA bundle (supports multiple OS types via env var)
       # Default fallback order: RHEL/Fedora -> Debian/Ubuntu -> macOS
       - ${HOST_CA_BUNDLE:-/etc/pki/tls/certs/ca-bundle.crt}:/etc/ssl/certs/ca-bundle.crt:ro
@@ -156,6 +157,7 @@ services:
       - GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/certs/spicedb-cert.pem
       # SSL cert file uses mounted path (same for all systems)
       - SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt
+      - KARTOGRAPH_EXTRACTION_SKILLS_DIR=/app/skills
     depends_on:
       postgres:
         condition: service_healthy

deploy/apps/kartograph/base/api-deployment.yaml

@@ -155,11 +155,15 @@ spec:
                 secretKeyRef:
                   name: kartograph-sso-client-swagger-docs
                   key: client_id
+            - name: KARTOGRAPH_EXTRACTION_SKILLS_DIR
+              value: /app/skills
 
           volumeMounts:
             - name: spicedb-ca
               mountPath: /etc/spicedb-ca
               readOnly: true
+            - name: extraction-skills
+              mountPath: /app/skills
           livenessProbe:
             httpGet:
               path: /health
@@ -190,3 +194,5 @@ spec:
             items:
               - key: service-ca.crt
                 path: service-ca.crt
+        - name: extraction-skills
+          emptyDir: {}

env/api.env

@@ -11,5 +11,6 @@ SPICEDB_PRESHARED_KEY="changeme"
 KARTOGRAPH_CORS_ORIGINS=["http://localhost:3000"]
 KARTOGRAPH_IAM_BOOTSTRAP_ADMIN_USERNAMES='["alice"]'
 KARTOGRAPH_IAM_SINGLE_TENANT_MODE=false
-# Generate with uv run python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
-KARTOGRAPH_MGMT_ENCRYPTION_KEY="vwN4rUcH-KL-UyJsL8hc6apftRUTovwec6L2M5uF5OE="
+KARTOGRAPH_MGMT_ENCRYPTION_KEY="vwN4rUcH-KL-UyJsL8hc6apftRUTovwec6L2M5uF5OE="
+KARTOGRAPH_EXTRACTION_RUNTIME_BACKEND=memory
+KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_ENGINE=auto

keycloak/realm.json

@@ -76,6 +76,7 @@
   ],
   "users": [
     {
+      "id": "91bd9b81-5c1d-4307-8dcd-3b80dcc68894",
       "username": "alice",
       "enabled": true,
       "email": "alice@example.com",
@@ -90,6 +91,7 @@
       ]
     },
     {
+      "id": "7ac7083e-42c8-4643-8b2f-052ffc579ea2",
       "username": "bob",
       "enabled": true,
       "email": "bob@example.com",

skills/subagent-delivery/SKILL.md

@@ -0,0 +1,206 @@
+---
+name: subagent-delivery
+description: >
+  Executes a GitHub issue end-to-end with consistent branch, test, PR, and merge behavior.
+  Use when implementing units of work with sub-agents, preparing pull requests, resolving merge
+  conflicts, or when the user asks to run issue-by-issue delivery into feature/manage-knowledge-graph.
+  Supports parallel delivery waves with explicit blocker-question escalation.
+---
+
+# Subagent Delivery Protocol
+
+Follow this protocol for every assigned issue.
+
+System prompt template for spawned Claude instances:
+
+- `skills/subagent-delivery/claude-instance-system-prompt.txt`
+
+## Parallel Execution Model
+
+Use this model whenever multiple issues are independent:
+
+1. One subagent per issue branch.
+2. Shared target branch: `feature/manage-knowledge-graph`.
+3. No shared working branch between agents.
+4. Each subagent works to PR-ready state independently.
+5. Merge in dependency order (foundational backend before UI polish when coupled).
+
+If two issues touch the same files heavily, either:
+- serialize those two issues, or
+- split scope so each agent owns non-overlapping symbols.
+
+## Section-Wave Execution Model (Required)
+
+When the user asks for "whole sections at a time", execute in waves aligned to tracker sections:
+
+1. **Section A: Core lifecycle/data**
+   - `#643 #644 #645 #646 #659 #660 #661 #662 #663`
+2. **Section B: Extraction runtime/session**
+   - `#649 #650 #651 #652 #653 #654`
+3. **Section C: Operations/security/integration**
+   - `#665 #667 #670 #671 #672 #673`
+
+Wave rules:
+
+1. Run independent issues in parallel with one Claude instance per issue.
+2. Respect dependencies inside the section (foundation issues first).
+3. Keep all PRs targeting `feature/manage-knowledge-graph`.
+4. Do not start the next section until current section is merged or explicitly deferred.
+5. For each section, maintain a live status board:
+   - `queued`, `in_progress`, `blocked`, `in_review`, `merged`
+
+## Scope and Inputs
+
+Before coding, gather:
+
+1. Issue number and acceptance criteria.
+2. Target branch: `feature/manage-knowledge-graph`.
+3. Current repository state (`git status`, `git branch -vv`).
+4. Context pack (required):
+   - relevant specs under `specs/`
+   - bounded context ownership (management/ingestion/extraction/graph/querying/ui)
+   - existing tests near touched code
+   - architectural constraints from `AGENTS.md`
+
+If acceptance criteria are ambiguous, ask one focused question before implementation.
+
+## Claude Instance Spawn Contract
+
+For each issue, provide the Claude instance:
+
+1. Issue ID + title + acceptance criteria summary.
+2. Branch naming requirement:
+   - `feat/issue-<id>-<short-scope>` or `fix/issue-<id>-<short-scope>`
+3. Required reads:
+   - `AGENTS.md`
+   - relevant `specs/*.spec.md`
+   - related tests in touched context
+4. TDD requirement:
+   - tests first, then implementation, then verification
+5. Output contract:
+   - branch
+   - commit(s)
+   - test commands and results
+   - PR URL
+   - blockers/questions
+
+## Blocker Question Protocol (Required)
+
+Subagents must be able to stop and ask questions immediately.
+
+Trigger a blocker question when any of these is true:
+
+1. More than one valid interpretation of acceptance criteria.
+2. Missing security/tenancy/authorization decision.
+3. Required external behavior is unspecified.
+4. You would otherwise make an irreversible guess.
+
+When blocked:
+
+1. Stop implementation at the decision boundary.
+2. Ask one concise question in the active agent chat immediately.
+3. Include:
+   - what is ambiguous
+   - 2-3 concrete options
+   - recommended option and why
+4. If working from a GitHub issue, mirror the same question as an issue comment so the orchestrator can batch unresolved questions across agents.
+5. Continue only non-blocked work; do not guess on blocked decisions.
+
+If a blocker impacts multiple active instances:
+
+1. Pause affected issues.
+2. Continue unaffected issues in parallel.
+3. Post one consolidated orchestrator decision update.
+4. Resume paused issues with explicit instruction delta.
+
+## Git Workflow
+
+1. Ensure local target branch is up to date:
+   - `git checkout feature/manage-knowledge-graph`
+   - `git pull --ff-only`
+2. Create a dedicated branch per issue:
+   - `feat/issue-<id>-<short-scope>` for features
+   - `fix/issue-<id>-<short-scope>` for fixes
+3. Never mix multiple issues in one branch.
+4. Keep commits atomic and conventional (`feat:`, `fix:`, `refactor:`, `test:`).
+
+## Implementation Workflow (TDD Required)
+
+1. Read relevant spec(s) and affected bounded context code first.
+2. Write/adjust tests for expected behavior before implementation.
+3. Implement minimal code to satisfy tests.
+4. Run focused tests first, then broader suite for touched context.
+5. Run lints/type checks for changed files when applicable.
+6. If behavior depends on configuration, use settings/DI instead of hardcoding.
+7. If new ambiguity appears mid-implementation, invoke the Blocker Question Protocol.
+
+## PR Workflow
+
+1. Push branch to origin with upstream tracking.
+2. Open PR against `feature/manage-knowledge-graph`.
+3. Use this body structure:
+
+```markdown
+## Summary
+- <what changed and why>
+- <important architectural/security note>
+
+## Testing
+- [x] <unit tests run>
+- [x] <integration tests run if applicable>
+- [ ] <manual verification if pending>
+
+## Risks
+- <none> or <known risk + mitigation>
+```
+
+4. Link the issue in PR body using `Closes #<id>` when appropriate.
+5. If any assumptions were made, include an explicit assumptions list in PR body.
+
+## Merge and Conflict Handling
+
+1. Before merge, ensure CI checks are green.
+2. If branch is stale, rebase or merge target branch cleanly.
+3. Resolve conflicts preserving:
+   - Spec-required behavior
+   - Existing user changes
+   - Authorization and tenancy boundaries
+4. Re-run tests after conflict resolution.
+5. Merge into `feature/manage-knowledge-graph` only after verification.
+
+## Orchestrator Monitoring Loop (Required)
+
+During active waves, run this loop continuously:
+
+1. Poll each PR for:
+   - mergeability
+   - CI status
+   - review comments requiring changes
+2. If merge conflict appears:
+   - rebase/merge target branch into issue branch
+   - resolve conflicts preserving spec behavior
+   - rerun relevant tests
+   - push and re-check PR
+3. If CI fails:
+   - fix in same issue branch
+   - do not move issue scope
+4. Update section status board and report progress to user.
+
+## Orchestrator Handoff Contract
+
+Each subagent must hand back:
+
+1. Branch name and PR URL.
+2. Test commands run with pass/fail status.
+3. Any unresolved questions (if still blocked).
+4. Any assumptions that were taken and why they are safe.
+
+## Non-Negotiables
+
+- Do not use destructive git commands.
+- Do not skip tests.
+- Do not disable hooks.
+- Do not commit secrets or credentials.
+- Prefer fakes over mocks in unit tests when testing domain/application behavior.
+- Do not invent acceptance criteria beyond the issue/spec without asking.
+