Add AuthRetrying state for transient OAuth errors

When an OAuth refresh fails as a transient error for longer than the
in-loop short retry (~1-2 minutes at defaults), the monitor used to
mark the workload unauthenticated and exit. The workload stayed dead
even after the underlying condition cleared on its own (e.g. a brief
VPN disconnect routing requests through a different network path).
Now the monitor keeps retrying on a configurable longer cadence
(default 10 minutes) until either success or a configurable ceiling
(default 24 hours), at which point the workload is finally marked
unauthenticated.

Fixes #5349

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Signed-off-by: Greg Katz <gkatz@indeed.com>

Author: gkatz2

cmd/thv/app/common.go

@@ -157,12 +157,14 @@ func completeLogsArgs(cmd *cobra.Command, args []string, _ string) ([]string, co
 }
 
 // workloadStatusIndicator returns the status string with a visual indicator prepended
-// for statuses that warrant user attention (unauthenticated, policy_stopped).
+// for statuses that warrant user attention (unauthenticated, auth_retrying, policy_stopped).
 // All other statuses are returned as plain strings.
 func workloadStatusIndicator(status runtime.WorkloadStatus) string {
 	switch status {
 	case runtime.WorkloadStatusUnauthenticated:
 		return "⚠️  " + string(status)
+	case runtime.WorkloadStatusAuthRetrying:
+		return "🔄 " + string(status)
 	case runtime.WorkloadStatusPolicyStopped:
 		return "🚫 " + string(status)
 	case runtime.WorkloadStatusRunning, runtime.WorkloadStatusStopped, runtime.WorkloadStatusError,

cmd/thv/app/common_test.go

@@ -4,9 +4,12 @@
 package app
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/spf13/cobra"
+
+	"github.com/stacklok/toolhive/pkg/container/runtime"
 )
 
 func TestAddFormatFlag(t *testing.T) {
@@ -266,3 +269,38 @@ func TestIsOIDCEnabled(t *testing.T) {
 		})
 	}
 }
+
+func TestWorkloadStatusIndicator(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name      string
+		status    runtime.WorkloadStatus
+		wantHas   string // substring that must appear
+		wantExact string // if non-empty, must match exactly
+	}{
+		{"unauthenticated has ⚠️ prefix", runtime.WorkloadStatusUnauthenticated, "⚠️", ""},
+		{"auth_retrying has 🔄 prefix", runtime.WorkloadStatusAuthRetrying, "🔄", ""},
+		{"policy_stopped has 🚫 prefix", runtime.WorkloadStatusPolicyStopped, "🚫", ""},
+		{"running passes through plain", runtime.WorkloadStatusRunning, "", "running"},
+		{"stopped passes through plain", runtime.WorkloadStatusStopped, "", "stopped"},
+		{"unhealthy passes through plain", runtime.WorkloadStatusUnhealthy, "", "unhealthy"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got := workloadStatusIndicator(tc.status)
+			if tc.wantExact != "" && got != tc.wantExact {
+				t.Errorf("workloadStatusIndicator(%q) = %q, want exact %q",
+					tc.status, got, tc.wantExact)
+			}
+			if tc.wantHas != "" && !strings.Contains(got, tc.wantHas) {
+				t.Errorf("workloadStatusIndicator(%q) = %q, want substring %q",
+					tc.status, got, tc.wantHas)
+			}
+			if !strings.Contains(got, string(tc.status)) {
+				t.Errorf("workloadStatusIndicator(%q) = %q, must include status name",
+					tc.status, got)
+			}
+		})
+	}
+}

cmd/thv/app/list.go

@@ -248,7 +248,7 @@ func printTextOutput(workloadList []core.Workload, upgrades map[string]*upgrade.
 
 	// Print workload information
 	for _, c := range workloadList {
-		// Highlight unauthenticated and policy-stopped workloads with indicators
+		// Highlight unauthenticated, auth-retrying, and policy-stopped workloads with indicators
 		status := workloadStatusIndicator(c.Status)
 
 		// Print workload information

cmd/thv/app/ui/styles.go

@@ -71,6 +71,9 @@ var (
 	pillUnauthed = lipgloss.NewStyle().
 			Background(bgWarning).Foreground(ColorYellow).
 			Padding(0, 1).Render("⚠ unauthed")
+	pillAuthRetrying = lipgloss.NewStyle().
+				Background(bgWarning).Foreground(ColorYellow).
+				Padding(0, 1).Render("🔄 retrying")
 
 	keyStyle  = lipgloss.NewStyle().Foreground(ColorDim2)
 	portStyle = lipgloss.NewStyle().Foreground(ColorCyan).Bold(true)
@@ -97,6 +100,8 @@ func RenderStatusDot(status rt.WorkloadStatus) string {
 		return dotWarning
 	case rt.WorkloadStatusUnauthenticated:
 		return dotWarning
+	case rt.WorkloadStatusAuthRetrying:
+		return dotWarning
 	case rt.WorkloadStatusRemoving:
 		return dotWarning
 	case rt.WorkloadStatusPolicyStopped:
@@ -128,6 +133,8 @@ func RenderStatusPill(status rt.WorkloadStatus) string {
 		return pillUnknown
 	case rt.WorkloadStatusUnauthenticated:
 		return pillUnauthed
+	case rt.WorkloadStatusAuthRetrying:
+		return pillAuthRetrying
 	case rt.WorkloadStatusPolicyStopped:
 		return pillStopped
 	default:

docs/arch/02-core-concepts.md

@@ -38,6 +38,7 @@ A **workload** is the fundamental deployment unit in ToolHive. It represents eve
 - `error` - Workload encountered an error
 - `unhealthy` - Workload is running but unhealthy
 - `unauthenticated` - Remote workload cannot authenticate (expired tokens)
+- `auth_retrying` - Remote workload's token refresh is failing transiently; monitor is still retrying until success (→ `running`) or the configured ceiling (→ `unauthenticated`)
 - `unknown` - Workload status cannot be determined
 - `policy_stopped` - Workload was stopped by policy enforcement
 

docs/arch/08-workloads-lifecycle.md

@@ -22,6 +22,7 @@ stateDiagram-v2
     Running --> Stopping: Stop
     Running --> Unhealthy: Health Failed
     Running --> Unauthenticated: Auth Failed
+    Running --> AuthRetrying: Transient Token Refresh Failures
     Running --> Stopped: Container Exit
 
     Stopping --> Stopped: Success
@@ -31,16 +32,27 @@ stateDiagram-v2
     Unauthenticated --> Starting: Re-authenticate
     Unauthenticated --> Removing: Delete
 
+    AuthRetrying --> Running: Refresh Succeeds
+    AuthRetrying --> Unauthenticated: Ceiling Exceeded or Permanent Error
+
     Removing --> [*]: Success
     Error --> Starting: Restart
     Error --> Removing: Delete
 ```
 
 **States**: `pkg/container/runtime/types.go`
 - `starting`, `running`, `stopping`, `stopped`
-- `removing`, `error`, `unhealthy`, `unauthenticated`
+- `removing`, `error`, `unhealthy`, `unauthenticated`, `auth_retrying`
 - `policy_stopped`, `unknown`
 
+The `auth_retrying` cadence and ceiling can be tuned via environment
+variables on the proxy process:
+
+- `TOOLHIVE_TOKEN_AUTH_RETRYING_TICK_INTERVAL` (default `10m`): cadence
+  between background refresh attempts during the AuthRetrying window.
+- `TOOLHIVE_TOKEN_AUTH_RETRYING_MAX_ELAPSED` (default `24h`): ceiling
+  before the workload is finally marked `unauthenticated`.
+
 ## Core Operations
 
 ### Deploy

docs/server/docs.go

@@ -33,7 +33,7 @@ type workloadListResponse struct {
 type workloadStatusResponse struct {
 	// Current status of the workload
 	//nolint:lll // enums tag needed for swagger generation with --parseDependencyLevel
-	Status runtime.WorkloadStatus `json:"status" enums:"running,stopped,error,starting,stopping,unhealthy,removing,unknown,unauthenticated,policy_stopped"`
+	Status runtime.WorkloadStatus `json:"status" enums:"running,stopped,error,starting,stopping,unhealthy,removing,unknown,unauthenticated,auth_retrying,policy_stopped"`
 }
 
 // updateRequest represents the request to update an existing workload