|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""CLI used by the integration tests to drive the snapshot-agent. |
| 3 | +
|
| 4 | +All snapshot/restore calls in the test suite go through this script so that |
| 5 | +the tests exercise the real production path: the Python client |
| 6 | +(timeslice.snapshot_agent) talking gRPC to the Go agent. Configs are |
| 7 | +constructed here, in Python, the same way a real workload would build them. |
| 8 | +
|
| 9 | +Usage: |
| 10 | + agentctl.py --agent HOST:PORT snapshot|restore --job-id ID --backend cuda|app |
| 11 | + [--pids 1,2,3] (cuda) |
| 12 | + [--app vllm|sglang] [--endpoints URL,URL] |
| 13 | + [--mode offload|discard] [--tags a,b] (app) |
| 14 | +
|
| 15 | +Exits 0 when the operation completes, 1 otherwise. |
| 16 | +""" |
| 17 | + |
| 18 | +import argparse |
| 19 | +import sys |
| 20 | + |
| 21 | +from timeslice.snapshot_agent import SnapshotAgentClient, snapshot_agent_pb2 |
| 22 | + |
| 23 | +APPS = { |
| 24 | + "vllm": snapshot_agent_pb2.APP_VLLM, |
| 25 | + "sglang": snapshot_agent_pb2.APP_SGLANG, |
| 26 | +} |
| 27 | + |
| 28 | +MODES = { |
| 29 | + "": snapshot_agent_pb2.SUSPEND_MODE_UNSPECIFIED, |
| 30 | + "offload": snapshot_agent_pb2.SUSPEND_MODE_OFFLOAD, |
| 31 | + "discard": snapshot_agent_pb2.SUSPEND_MODE_DISCARD, |
| 32 | +} |
| 33 | + |
| 34 | + |
| 35 | +def build_config(args: argparse.Namespace) -> snapshot_agent_pb2.BackendConfig: |
| 36 | + if args.backend == "cuda": |
| 37 | + cuda = snapshot_agent_pb2.CudaBackendConfig() |
| 38 | + if args.pids: |
| 39 | + cuda.explicit_target.pids.extend(int(p) for p in args.pids.split(",")) |
| 40 | + return snapshot_agent_pb2.BackendConfig(cuda=cuda) |
| 41 | + |
| 42 | + if args.backend == "app": |
| 43 | + if args.app not in APPS: |
| 44 | + raise ValueError(f"--app is required for --backend app (one of {sorted(APPS)})") |
| 45 | + app_endpoint = snapshot_agent_pb2.AppEndpointConfig( |
| 46 | + app=APPS[args.app], |
| 47 | + endpoints=args.endpoints.split(",") if args.endpoints else [], |
| 48 | + mode=MODES[args.mode], |
| 49 | + ) |
| 50 | + if args.tags: |
| 51 | + app_endpoint.tags.extend(args.tags.split(",")) |
| 52 | + return snapshot_agent_pb2.BackendConfig(app_endpoint=app_endpoint) |
| 53 | + |
| 54 | + raise ValueError(f"unknown backend {args.backend!r}") |
| 55 | + |
| 56 | + |
| 57 | +def main() -> int: |
| 58 | + parser = argparse.ArgumentParser(description=__doc__) |
| 59 | + parser.add_argument("action", choices=["snapshot", "restore"]) |
| 60 | + parser.add_argument("--agent", required=True, help="agent endpoint HOST:PORT") |
| 61 | + parser.add_argument("--job-id", required=True) |
| 62 | + parser.add_argument("--group", default="test") |
| 63 | + parser.add_argument("--backend", required=True, choices=["cuda", "app"]) |
| 64 | + parser.add_argument("--pids", default="", help="comma-separated PIDs (cuda)") |
| 65 | + parser.add_argument("--app", default="", choices=["", "vllm", "sglang"], help="application (app backend)") |
| 66 | + parser.add_argument("--endpoints", default="", help="comma-separated application URLs") |
| 67 | + parser.add_argument("--mode", default="", choices=["", "offload", "discard"], help="suspend mode") |
| 68 | + parser.add_argument("--tags", default="", help="comma-separated region tags") |
| 69 | + args = parser.parse_args() |
| 70 | + |
| 71 | + config = build_config(args) |
| 72 | + |
| 73 | + with SnapshotAgentClient(args.agent) as client: |
| 74 | + if args.action == "snapshot": |
| 75 | + result = client.snapshot_and_wait(args.job_id, args.group, backend_config=config) |
| 76 | + else: |
| 77 | + result = client.restore_and_wait(args.job_id, args.group, backend_config=config) |
| 78 | + |
| 79 | + if result.status != "OPERATION_STATUS_COMPLETE": |
| 80 | + print( |
| 81 | + f"{args.action} {args.job_id} finished with {result.status}: {result.error}", |
| 82 | + file=sys.stderr, |
| 83 | + ) |
| 84 | + return 1 |
| 85 | + |
| 86 | + print(f"{args.action} {args.job_id} complete in {result.elapsed_ms}ms") |
| 87 | + return 0 |
| 88 | + |
| 89 | + |
| 90 | +if __name__ == "__main__": |
| 91 | + sys.exit(main()) |
0 commit comments