feat(toolkit): add REST healthcheck infrastructure

gears/system/api-gateway/src/gear.rs

@@ -77,6 +77,8 @@ pub struct ApiGateway {
     pub(crate) final_router: Mutex<Option<axum::Router>>,
     // AuthN Resolver client (resolved during init, None when auth_disabled)
     pub(crate) authn_client: Mutex<Option<Arc<dyn AuthNResolverClient>>>,
+    // Readiness healthcheck registry — populated during REST wiring phase
+    pub(crate) healthcheck_registry: Mutex<Option<Arc<toolkit::RestHealthcheckRegistry>>>,
 
     // Duplicate detection (per (method, path) and per handler id)
     pub(crate) registered_routes: DashMap<(Method, String), ()>,
@@ -92,24 +94,51 @@ impl Default for ApiGateway {
             router_cache: RouterCache::new(default_router),
             final_router: Mutex::new(None),
             authn_client: Mutex::new(None),
+            healthcheck_registry: Mutex::new(None),
             registered_routes: DashMap::new(),
             registered_handlers: DashMap::new(),
         }
     }
 }
 
 impl ApiGateway {
-    fn apply_prefix_nesting(mut router: Router, prefix: &str) -> Router {
+    fn apply_prefix_nesting(
+        &self,
+        router: Router,
+        prefix: &str,
+        hc_registry: Arc<toolkit::RestHealthcheckRegistry>,
+        authn_client: Option<Arc<dyn AuthNResolverClient>>,
+    ) -> Result<Router> {
         if prefix.is_empty() {
-            return router;
+            return Ok(router);
         }
 
-        let top = Router::new()
-            .route("/health", get(web::health_check))
-            .route("/healthz", get(|| async { "ok" }));
+        // Root-mounted health routes must go through the same middleware stack
+        // (request ID, tracing, metrics, CatchPanicLayer) as the prefixed API.
+        let top = self.apply_middleware_stack(Self::health_routes(hc_registry), authn_client)?;
+        let nested = Router::new().nest(prefix, router);
+        Ok(top.merge(nested))
+    }
 
-        router = Router::new().nest(prefix, router);
-        top.merge(router)
+    fn health_routes(hc_registry: Arc<toolkit::RestHealthcheckRegistry>) -> Router {
+        let reg_h = hc_registry.clone();
+        let reg_r = hc_registry;
+        Router::new()
+            .route(
+                "/health",
+                get(move || {
+                    let r = reg_h.clone();
+                    async move { web::health_detail(r).await }
+                }),
+            )
+            .route("/healthz", get(|| async { "ok" }))
+            .route(
+                "/readyz",
+                get(move || {
+                    let r = reg_r.clone();
+                    async move { web::readyz_check(r).await }
+                }),
+            )
     }
 
     /// Create a new `ApiGateway` instance with the given configuration
@@ -122,6 +151,7 @@ impl ApiGateway {
             router_cache: RouterCache::new(default_router),
             final_router: Mutex::new(None),
             authn_client: Mutex::new(None),
+            healthcheck_registry: Mutex::new(None),
             registered_routes: DashMap::new(),
             registered_handlers: DashMap::new(),
         }
@@ -160,6 +190,7 @@ impl ApiGateway {
         // Always mark built-in health check routes as public
         public_routes.insert((Method::GET, "/health".to_owned()));
         public_routes.insert((Method::GET, "/healthz".to_owned()));
+        public_routes.insert((Method::GET, "/readyz".to_owned()));
 
         public_routes.insert((Method::GET, "/docs".to_owned()));
         public_routes.insert((Method::GET, "/openapi.json".to_owned()));
@@ -484,19 +515,23 @@ impl ApiGateway {
         }
 
         tracing::debug!("Building new router (standalone/fallback mode)");
-        // In standalone mode (no REST pipeline), register both health endpoints here.
+        // In standalone mode (no REST pipeline), register health endpoints here.
         // In normal operation, rest_prepare() registers these instead.
-        let mut router = Router::new()
-            .route("/health", get(web::health_check))
-            .route("/healthz", get(|| async { "ok" }));
+        let hc_registry = self
+            .healthcheck_registry
+            .lock()
+            .clone()
+            .unwrap_or_else(|| Arc::new(toolkit::RestHealthcheckRegistry::new()));
+
+        let mut router = Self::health_routes(hc_registry.clone());
 
         // Apply all middleware layers including auth, above the router
         let authn_client = self.authn_client.lock().clone();
-        router = self.apply_middleware_stack(router, authn_client)?;
+        router = self.apply_middleware_stack(router, authn_client.clone())?;
 
         let config = self.get_cached_config();
         let prefix = Self::normalize_prefix_path(&config.prefix_path)?;
-        router = Self::apply_prefix_nesting(router, &prefix);
+        router = self.apply_prefix_nesting(router, &prefix, hc_registry, authn_client)?;
 
         // Cache the built router for future use
         self.router_cache.store(router.clone());
@@ -706,17 +741,24 @@ impl toolkit::Gear for ApiGateway {
 impl toolkit::contracts::ApiGatewayCapability for ApiGateway {
     fn rest_prepare(
         &self,
-        _ctx: &toolkit::context::GearCtx,
+        ctx: &toolkit::context::GearCtx,
         router: axum::Router,
     ) -> anyhow::Result<axum::Router> {
-        // Add health check endpoints:
-        // - /health: detailed JSON response with status and timestamp
-        // - /healthz: simple "ok" liveness probe (Kubernetes-style)
-        let router = router
-            .route("/health", get(web::health_check))
-            .route("/healthz", get(|| async { "ok" }));
-
-        // You may attach global middlewares here (trace, compression, cors), but do not start server.
+        // Retrieve the healthcheck registry created in run_rest_phase before this call.
+        // Falls back to an empty registry so standalone tests still compile.
+        let hc_registry = ctx
+            .client_hub()
+            .get::<toolkit::RestHealthcheckRegistry>()
+            .unwrap_or_else(|_| Arc::new(toolkit::RestHealthcheckRegistry::new()));
+
+        *self.healthcheck_registry.lock() = Some(hc_registry.clone());
+
+        // Add health endpoints:
+        // - /healthz: shallow liveness probe, never runs user checks
+        // - /readyz: readiness probe, runs user REST healthchecks
+        // - /health: detailed readiness JSON with per-component status
+        let router = router.merge(Self::health_routes(hc_registry));
+
         tracing::debug!("REST host prepared base router with health check endpoints");
         Ok(router)
     }
@@ -735,10 +777,16 @@ impl toolkit::contracts::ApiGatewayCapability for ApiGateway {
         // Apply middleware stack (including auth) to the final router
         tracing::debug!("Applying middleware stack to finalized router");
         let authn_client = self.authn_client.lock().clone();
-        router = self.apply_middleware_stack(router, authn_client)?;
+        router = self.apply_middleware_stack(router, authn_client.clone())?;
+
+        let hc_registry = self
+            .healthcheck_registry
+            .lock()
+            .clone()
+            .unwrap_or_else(|| Arc::new(toolkit::RestHealthcheckRegistry::new()));
 
         let prefix = Self::normalize_prefix_path(&config.prefix_path)?;
-        router = Self::apply_prefix_nesting(router, &prefix);
+        router = self.apply_prefix_nesting(router, &prefix, hc_registry, authn_client)?;
 
         // Keep the finalized router to be used by `serve()`
         *self.final_router.lock() = Some(router.clone());

gears/system/api-gateway/src/web.rs

@@ -1,10 +1,15 @@
 use axum::{
     http::StatusCode,
-    response::{Html, Json},
+    response::{Html, IntoResponse, Json, Response},
     routing::{MethodRouter, get},
 };
 use chrono::{SecondsFormat, Utc};
-use serde_json::{Value, json};
+use serde_json::json;
+use std::sync::Arc;
+use std::time::Duration;
+use toolkit::RestHealthcheckRegistry;
+
+const HEALTHCHECK_TIMEOUT: Duration = Duration::from_millis(500);
 
 /// Returns a 501 Not Implemented handler for operations without implementations
 #[allow(dead_code)]
@@ -20,11 +25,36 @@ pub fn placeholder_handler_501() -> MethodRouter {
     })
 }
 
-pub async fn health_check() -> Json<Value> {
-    Json(json!({
-        "status": "healthy",
-        "timestamp": Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true)
-    }))
+/// Detailed readiness JSON endpoint (`GET /health`).
+///
+/// Runs all registered REST healthchecks and returns a JSON report.
+/// Returns 200 (Healthy or Degraded), 503 (Unhealthy).
+pub async fn health_detail(registry: Arc<RestHealthcheckRegistry>) -> Response {
+    let report = registry.report(HEALTHCHECK_TIMEOUT).await;
+    let status_code = status_for_report(report.is_ready());
+    let body = Json(json!({
+        "status": report.status,
+        "timestamp": Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true),
+        "components": report.components,
+    }));
+    (status_code, body).into_response()
+}
+
+/// Readiness probe (`GET /readyz`).
+///
+/// Deep check: runs all registered REST healthchecks.
+/// Returns 200 (Healthy or Degraded), 503 (Unhealthy).
+pub async fn readyz_check(registry: Arc<RestHealthcheckRegistry>) -> StatusCode {
+    let report = registry.report(HEALTHCHECK_TIMEOUT).await;
+    status_for_report(report.is_ready())
+}
+
+fn status_for_report(report_ready: bool) -> StatusCode {
+    if report_ready {
+        StatusCode::OK
+    } else {
+        StatusCode::SERVICE_UNAVAILABLE
+    }
 }
 
 #[cfg(not(feature = "embed_elements"))]