feat(ui): KG-scoped data source onboarding (k-extract flow)

.gitignore

@@ -173,6 +173,8 @@ src/dev-ui/.output/
 
 certs/
 .instances/
+.kartograph/backups/
+.kartograph/kg-backups/
 
 # Demo web interface (not for production)
 demo-web/

AGENTS.md

@@ -127,9 +127,30 @@ must be triggered explicitly.
 **Standard (single developer):**
 ```bash
 make dev    # starts all services: Postgres, SpiceDB, Keycloak, API, Dev UI
-make down   # tears everything down
+make down   # stops containers; Postgres data volume is preserved
 ```
 
+**Dev data backup / restore** (knowledge graphs, ontology, graph data, IAM, SpiceDB):
+
+```bash
+make dev-backup        # snapshot DBs to .kartograph/backups/<timestamp>/
+make dev-backup-list   # list available snapshots
+make dev-restore       # restore the latest snapshot (prompts for confirmation)
+make dev-restore BACKUP=2026-06-12T20-10-33Z
+make dev-repair-age-graphs   # fix corrupt AGE graphs without full DB restore
+```
+
+For isolated instances, set the compose project name:
+
+```bash
+COMPOSE_PROJECT=kg-my-feature ./scripts/dev-data-backup.sh backup
+COMPOSE_PROJECT=kg-my-feature ./scripts/dev-data-backup.sh restore latest --yes
+```
+
+Avoid `docker compose down -v` unless you intend to wipe volumes. After a
+restore, if the dev UI shows an empty tenant, delete `~/.kartograph/token.json`
+and sign in again.
+
 **Isolated instance (agents / worktrees):**
 
 When working in a worktree or running multiple instances in parallel,

Makefile

@@ -23,8 +23,10 @@ certs:
 .PHONY: dev
 dev: certs
 	@echo "🧰 [Development] Starting application containers..."
+	@./scripts/cleanup-openshell-sandboxes.sh
+	docker compose -f compose.yaml -f compose.dev.yaml --profile build-only build agent-runtime
 	docker compose -f compose.yaml build
-	docker compose -f compose.yaml -f compose.dev.yaml --profile ui up -d
+	HOST_UID=$$(id -u) HOST_GID=$$(id -g) docker compose -f compose.yaml -f compose.dev.yaml --profile ui up -d
 	@echo "Done."
 	@echo "----------------------------"
 	@echo "API Root: http://localhost:8000"
@@ -35,6 +37,37 @@ dev: certs
 .PHONY: down
 down:
 	docker compose -f compose.yaml -f compose.dev.yaml down
+	@echo "Stopping Graph Management sticky, worker, and extraction job containers..."
+	-@docker ps -aq --filter name=kartograph-sticky- | xargs -r docker rm -f
+	-@docker ps -aq --filter name=kartograph-worker- | xargs -r docker rm -f
+	-@docker ps -aq --filter name=kartograph-extract- | xargs -r docker rm -f
+	-@./scripts/cleanup-openshell-sandboxes.sh
+
+.PHONY: dev-backup dev-restore dev-backup-list dev-repair-age-graphs
+dev-backup:
+	@./scripts/dev-data-backup.sh backup
+
+dev-restore:
+	@./scripts/dev-data-backup.sh restore $(or $(BACKUP),latest)
+
+dev-backup-list:
+	@./scripts/dev-data-backup.sh list
+
+dev-repair-age-graphs:
+	@./scripts/dev-data-backup.sh repair
+
+.PHONY: kg-backup kg-restore kg-backup-list
+kg-backup:
+	@test -n "$(KG_ID)" || (echo "Usage: make kg-backup KG_ID=<knowledge-graph-id>" && exit 1)
+	@./scripts/kg-data-backup.sh capture "$(KG_ID)"
+
+kg-restore:
+	@test -n "$(KG_ID)" || (echo "Usage: make kg-restore KG_ID=<knowledge-graph-id> [BACKUP=latest] [YES=1] [REPLACE=1]" && exit 1)
+	@./scripts/kg-data-backup.sh restore "$(KG_ID)" $(or $(BACKUP),latest) $(if $(YES),--yes,) $(if $(REPLACE),--replace,)
+
+kg-backup-list:
+	@test -n "$(KG_ID)" || (echo "Usage: make kg-backup-list KG_ID=<knowledge-graph-id>" && exit 1)
+	@./scripts/kg-data-backup.sh list "$(KG_ID)"
 
 
 .PHONY: run

compose.dev.yaml

@@ -1,19 +1,85 @@
 # Development overrides for compose.yaml
 services:
+  agent-runtime:
+    build:
+      context: ./src/agent-runtime
+      dockerfile: Dockerfile
+    image: kartograph-agent-runtime:dev
+    profiles: [ "build-only" ]
+
   api:
-    # Run as root in dev to handle host file permissions (any umask)
-    user: "${UID}:${GID}"
+    # Root required for Docker-out-of-Docker via mounted /var/run/docker.sock in dev
+    user: "0:0"
     environment:
       UV_CACHE_DIR: /tmp/uv-cache
+      HOST_UID: ${HOST_UID}
+      HOST_GID: ${HOST_GID}
+      KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_ENGINE: auto
+      KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_NETWORK: kartograph_kartograph
+      KARTOGRAPH_EXTRACTION_RUNTIME_STICKY_IMAGE: kartograph-agent-runtime:dev
+      KARTOGRAPH_EXTRACTION_RUNTIME_API_BASE_URL: http://api:8000
+      KARTOGRAPH_EXTRACTION_RUNTIME_WORKLOAD_TOKEN_SIGNING_KEY: kartograph-dev-workload-token-signing-key
+      KARTOGRAPH_EXTRACTION_RUNTIME_JOB_PACKAGE_WORK_DIR: /tmp/kartograph/job_packages
+      KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_RUN_UID: ${HOST_UID}
+      KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_RUN_GID: ${HOST_GID}
+      KARTOGRAPH_EXTRACTION_RUNTIME_STICKY_TURN_TIMEOUT_SECONDS: "3600"
+      KARTOGRAPH_EXTRACTION_RUNTIME_STICKY_MAX_TURNS: "500"
+      KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_HARDENING_ENABLED: "true"
+      ## Docker (Track A):
+      # KARTOGRAPH_EXTRACTION_RUNTIME_BACKEND: container
+      # KARTOGRAPH_EXTRACTION_RUNTIME_JOB_RUNNER: agentic_ci
+      ## OpenShell (Track B): uncomment backend/job_runner after `openshell gateway add` on the host
+      KARTOGRAPH_EXTRACTION_RUNTIME_BACKEND: openshell
+      KARTOGRAPH_EXTRACTION_RUNTIME_JOB_RUNNER: openshell
+      ## End Track A / Track B selection
+      KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_GATEWAY_URL: https://host.docker.internal:17670
+      # Forwards run inside this container; bind to 127.0.0.1 here, not on the host.
+      KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_RUNTIME_HOST: 127.0.0.1
+      KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_API_BASE_URL: http://host.docker.internal:8000
+      KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_POLICY_DIR: /etc/openshell/policies
+      KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_GATEWAY_NAME: openshell
+      # OpenShell CLI in the API container reads host gateway registration + mTLS from here
+      KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_XDG_CONFIG_HOME: /root/.config
+      # Host gateway must listen beyond localhost — in ~/.config/openshell/gateway.toml:
+      #   bind_address = "0.0.0.0:17670"
+      # then: systemctl --user restart openshell-gateway
+      KARTOGRAPH_EXTRACTION_RUNTIME_AGENTIC_CI_IMAGE: ghcr.io/opendatahub-io/ai-helpers:latest
+      # OpenShell extraction jobs use agentic-ci claude-sandbox (not ai-helpers or sticky runtime).
+      KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_EXTRACTION_IMAGE: quay.io/aipcc/agentic-ci/claude-sandbox:latest
+      KARTOGRAPH_EXTRACTION_RUNTIME_AGENTIC_CI_HARNESS: claude-code
+      KARTOGRAPH_EXTRACTION_RUNTIME_EXTRACTION_JOB_WORK_DIR: /tmp/kartograph/extraction_jobs
+      # Vertex AI for Claude Agent SDK in sticky assistant containers
+      CLAUDE_CODE_USE_VERTEX: "1"
+      ANTHROPIC_VERTEX_PROJECT_ID: itpc-gcp-hcm-pe-eng-claude
+      CLOUD_ML_REGION: us-east5
+      KARTOGRAPH_GCLOUD_CONFIG_MOUNT: ${HOME}/.config/gcloud
     volumes:
       # Mount the entire app directory  (minus  venv) for hot-reload
       - ./src/api:/app:z
       - /app/.venv
+      # Shared with sibling sticky containers launched via the host Docker socket
+      - /tmp/kartograph/job_packages:/tmp/kartograph/job_packages
+      - /tmp/kartograph/extraction_jobs:/tmp/kartograph/extraction_jobs
+      # gcloud ADC for Vertex-backed agentic-ci extraction job containers
+      - ${HOME}/.config/gcloud:${HOME}/.config/gcloud:ro,z
+      # Allow API process to launch sibling extraction runtime containers locally
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Docker/Podman CLI from host (required for container runtime backend)
+      - ${DOCKER_BIN:-/usr/bin/docker}:/usr/bin/docker:ro
+      # OpenShell CLI + mTLS config (host gateway; API container invokes openshell subprocess)
+      - /usr/bin/openshell:/usr/bin/openshell:ro
+      - ${HOME}/.config/openshell:/root/.config/openshell:ro,z
+      # forward start -d writes PID/state here; read-only parent mount hangs the CLI
+      - openshell-forwards:/root/.config/openshell/forwards
+      # OpenShell policy templates (Phase 3) when backend=openshell
+      - ./src/api/extraction/infrastructure/openshell/policies:/etc/openshell/policies:ro,z
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     command:
       - /bin/bash
       - -c
       - |
-        uv run fastapi dev main.py --host 0.0.0.0 --port 8000
+        uv sync --frozen && uv run alembic upgrade head && uv run fastapi dev main.py --host 0.0.0.0 --port 8000
 
   dev-ui:
     build:
@@ -30,7 +96,10 @@ services:
       - /app/.output
     environment:
       - HOST=0.0.0.0
-    command: ["pnpm", "run", "dev"]
+    command: [ "pnpm", "run", "dev" ]
     ports:
       - "3000:3000"
       - "24678:24678"
+
+volumes:
+  openshell-forwards:

compose.yaml

@@ -147,7 +147,7 @@ services:
       - ./certs:/certs:ro
       # Mount host CA bundle (supports multiple OS types via env var)
       # Default fallback order: RHEL/Fedora -> Debian/Ubuntu -> macOS
-      - ${HOST_CA_BUNDLE:-/etc/pki/tls/certs/ca-bundle.crt}:/etc/ssl/certs/ca-bundle.crt:ro
+      - ${HOST_CA_BUNDLE:-/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem}:/etc/ssl/certs/ca-bundle.crt:ro
     tty: true
     extra_hosts:
       - "localhost:host-gateway"

deploy/apps/kartograph/base/api-deployment.yaml

@@ -155,11 +155,44 @@ spec:
                 secretKeyRef:
                   name: kartograph-sso-client-swagger-docs
                   key: client_id
+            - name: KARTOGRAPH_EXTRACTION_RUNTIME_BACKEND
+              valueFrom:
+                configMapKeyRef:
+                  name: kartograph-config
+                  key: KARTOGRAPH_EXTRACTION_RUNTIME_BACKEND
+                  optional: true
+            - name: KARTOGRAPH_EXTRACTION_RUNTIME_API_BASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: kartograph-config
+                  key: KARTOGRAPH_EXTRACTION_RUNTIME_API_BASE_URL
+                  optional: true
+            - name: KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_HARDENING_ENABLED
+              valueFrom:
+                configMapKeyRef:
+                  name: kartograph-config
+                  key: KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_HARDENING_ENABLED
+                  optional: true
+            - name: KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_POLICY_DIR
+              valueFrom:
+                configMapKeyRef:
+                  name: kartograph-config
+                  key: KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_POLICY_DIR
+                  optional: true
+            - name: KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_POLICY_ENFORCEMENT
+              valueFrom:
+                configMapKeyRef:
+                  name: kartograph-config
+                  key: KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_POLICY_ENFORCEMENT
+                  optional: true
 
           volumeMounts:
             - name: spicedb-ca
               mountPath: /etc/spicedb-ca
               readOnly: true
+            - name: openshell-policies
+              mountPath: /etc/openshell/policies
+              readOnly: true
           livenessProbe:
             httpGet:
               path: /health
@@ -190,3 +223,6 @@ spec:
             items:
               - key: service-ca.crt
                 path: service-ca.crt
+        - name: openshell-policies
+          configMap:
+            name: kartograph-openshell-policies

deploy/apps/kartograph/base/configmap.yaml

@@ -22,3 +22,12 @@ data:
   DEV_UI_KEYCLOAK_URL: "http://keycloak:8080"
   DEV_UI_KEYCLOAK_REALM: "kartograph"
   DEV_UI_KEYCLOAK_CLIENT_ID: "kartograph-ui"
+  # Extraction runtime (container backend with Phase 0 hardening; switch to openshell in overlay)
+  KARTOGRAPH_EXTRACTION_RUNTIME_BACKEND: "container"
+  KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_NETWORK: "kartograph"
+  KARTOGRAPH_EXTRACTION_RUNTIME_API_BASE_URL: "http://kartograph-api:8000"
+  KARTOGRAPH_EXTRACTION_RUNTIME_AGENTIC_CI_API_BASE_URL: "http://kartograph-api:8000"
+  KARTOGRAPH_EXTRACTION_RUNTIME_CONTAINER_HARDENING_ENABLED: "true"
+  KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_POLICY_DIR: "/etc/openshell/policies"
+  KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_POLICY_ENFORCEMENT: "hard_requirement"
+  KARTOGRAPH_EXTRACTION_RUNTIME_OPENSHELL_RUNTIME_HOST: "127.0.0.1"

deploy/apps/kartograph/base/kustomization.yaml

@@ -28,6 +28,8 @@ resources:
   - configmap.yaml
   - spicedb-schema-configmap.yaml
   - spicedb-ca-configmap.yaml
+  - networkpolicy-sticky-runtime.yaml
+  - openshell-policies-configmap.yaml
 
 commonLabels:
   app.kubernetes.io/name: kartograph

deploy/apps/kartograph/base/networkpolicy-sticky-runtime.yaml

@@ -0,0 +1,33 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: kartograph-sticky-runtime-egress
+  labels:
+    app.kubernetes.io/component: sticky-runtime
+spec:
+  podSelector:
+    matchLabels:
+      kartograph.runtime.kind: sticky
+  policyTypes:
+    - Egress
+  egress:
+    # Kartograph workload API (in-cluster DNS name)
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/component: api
+      ports:
+        - protocol: TCP
+          port: 8000
+    # DNS resolution inside the cluster
+    - to:
+        - namespaceSelector: {}
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    # Vertex / Anthropic inference via OpenShell inference.local routing when enabled
+    - ports:
+        - protocol: TCP
+          port: 443

deploy/apps/kartograph/base/openshell-policies-configmap.yaml

@@ -0,0 +1,76 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kartograph-openshell-policies
+  labels:
+    app.kubernetes.io/component: openshell
+data:
+  gma-sticky-base.yaml: |
+    version: 1
+    name: gma-sticky-base
+    enforcement: hard_requirement
+    endpoints:
+      - "kartograph-api:8000:read-write"
+      - "inference.local:443:read-write"
+      - "aiplatform.googleapis.com:443:read-write"
+      - "*.aiplatform.googleapis.com:443:read-write"
+      - "oauth2.googleapis.com:443:read-write"
+    l7_allowed_paths:
+      - "/extraction/workloads/*"
+  gma-initial-schema-design.yaml: |
+    version: 1
+    name: gma-initial-schema-design
+    enforcement: hard_requirement
+    endpoints:
+      - "kartograph-api:8000:read-write"
+      - "inference.local:443:read-write"
+      - "aiplatform.googleapis.com:443:read-write"
+      - "*.aiplatform.googleapis.com:443:read-write"
+      - "oauth2.googleapis.com:443:read-write"
+    l7_allowed_paths:
+      - "/extraction/workloads/schema/*"
+      - "/extraction/workloads/graph/*"
+  gma-extraction-jobs.yaml: |
+    version: 1
+    name: gma-extraction-jobs
+    enforcement: hard_requirement
+    endpoints:
+      - "kartograph-api:8000:read-write"
+      - "inference.local:443:read-write"
+      - "aiplatform.googleapis.com:443:read-write"
+      - "*.aiplatform.googleapis.com:443:read-write"
+      - "oauth2.googleapis.com:443:read-write"
+    l7_allowed_paths:
+      - "/extraction/workloads/jobs/*"
+      - "/extraction/workloads/schema/*"
+      - "/extraction/workloads/graph/*"
+  gma-one-off-mutations.yaml: |
+    version: 1
+    name: gma-one-off-mutations
+    enforcement: hard_requirement
+    endpoints:
+      - "kartograph-api:8000:read-write"
+      - "inference.local:443:read-write"
+      - "aiplatform.googleapis.com:443:read-write"
+      - "*.aiplatform.googleapis.com:443:read-write"
+      - "oauth2.googleapis.com:443:read-write"
+    l7_allowed_paths:
+      - "/extraction/workloads/mutations/*"
+      - "/extraction/workloads/graph/*"
+  extraction-job.yaml: |
+    version: 1
+    name: extraction-job
+    enforcement: hard_requirement
+    endpoints:
+      - "kartograph-api:8000:read-write"
+      - "inference.local:443:read-write"
+      - "github.com:443:full"
+      - "*.github.com:443:full"
+      - "pypi.org:443:read-only"
+      - "files.pythonhosted.org:443:read-only"
+      - "aiplatform.googleapis.com:443:read-write"
+      - "*.aiplatform.googleapis.com:443:read-write"
+      - "oauth2.googleapis.com:443:read-write"
+      - "api.anthropic.com:443:read-write"
+    l7_allowed_paths:
+      - "/extraction/workloads/*"