feat: auto failover for API with LK Cloud

.github/workflows/test-api.yml

@@ -0,0 +1,60 @@
+# Copyright 2026 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Test API
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  failover:
+    runs-on: ubuntu-latest
+    services:
+      mock-server:
+        image: livekit/test-server:latest
+        ports:
+          - 9999:9999
+          - 10000:10000
+          - 10001:10001
+          - 10002:10002
+    steps:
+      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6
+
+      - name: Set up Opus
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsoxr-dev libopus-dev libopusfile-dev
+
+      - name: Set up Go
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
+        with:
+          go-version: 1.26.0
+
+      - name: Wait for mock server
+        run: |
+          for i in $(seq 1 30); do
+            curl -sf http://127.0.0.1:9999/settings/regions >/dev/null && exit 0
+            sleep 1
+          done
+          echo "mock server did not become ready" && exit 1
+
+      - name: Run API tests
+        run: go test -v -count=1 ./apitest/...

agent_client.go

@@ -29,7 +29,7 @@ func NewAgentClient(url string, apiKey string, apiSecret string, opts ...AgentCl
 	}
 	c := &AgentClient{
 		authBase:   authBase{apiKey, apiSecret},
-		httpClient: &http.Client{},
+		httpClient: newAPIHTTPClient(),
 	}
 	for _, opt := range opts {
 		opt(c)

agent_dispatch_client.go

@@ -16,7 +16,6 @@ package lksdk
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/twitchtv/twirp"
 
@@ -31,7 +30,7 @@ type AgentDispatchClient struct {
 
 func NewAgentDispatchServiceClient(url string, apiKey string, secretKey string, opts ...twirp.ClientOption) *AgentDispatchClient {
 	url = signalling.ToHttpURL(url)
-	client := livekit.NewAgentDispatchServiceProtobufClient(url, &http.Client{}, opts...)
+	client := livekit.NewAgentDispatchServiceProtobufClient(url, newAPIHTTPClient(), opts...)
 
 	return &AgentDispatchClient{
 		agentDispatchService: client,

agent_simulation_client.go

@@ -16,7 +16,6 @@ package lksdk
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/twitchtv/twirp"
 
@@ -29,7 +28,7 @@ type AgentSimulationClient struct {
 }
 
 func NewAgentSimulationClient(url string, apiKey string, apiSecret string, opts ...twirp.ClientOption) *AgentSimulationClient {
-	client := livekit.NewAgentSimulationProtobufClient(url, &http.Client{}, opts...)
+	client := livekit.NewAgentSimulationProtobufClient(url, newAPIHTTPClient(), opts...)
 	return &AgentSimulationClient{
 		simulationClient: client,
 		authBase:         authBase{apiKey, apiSecret},

apitest/failover_test.go

@@ -0,0 +1,139 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package apitest holds black-box API tests that exercise the public server SDK
+// against the shared mock LiveKit API server (livekit/livekit cmd/test-server).
+// Point them at a running instance with LK_TEST_SERVER_URL (default
+// http://127.0.0.1:9999); they skip when no server is reachable. In CI the
+// server is booted as a Docker container.
+//
+// See cmd/test-server/README.md for the X-Lk-Mock-* control protocol. Mock
+// directives are passed to the SDK via twirp request headers, which the
+// failover transport forwards to the discovery fetch and every retry.
+package apitest
+
+import (
+	"context"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"github.com/twitchtv/twirp"
+
+	"github.com/livekit/protocol/livekit"
+	lksdk "github.com/livekit/server-sdk-go/v2"
+)
+
+const (
+	hdrFailRegions = "X-Lk-Mock-Fail-Regions"
+	hdrFailMode    = "X-Lk-Mock-Fail-Mode"
+	hdrFailStatus  = "X-Lk-Mock-Fail-Status"
+	hdrRegionsStat = "X-Lk-Mock-Regions-Status"
+)
+
+func testServerURL(t *testing.T) string {
+	url := os.Getenv("LK_TEST_SERVER_URL")
+	if url == "" {
+		url = "http://127.0.0.1:9999"
+	}
+	resp, err := http.Get(url + "/settings/regions")
+	if err != nil {
+		t.Skipf("mock test server not reachable at %s (set LK_TEST_SERVER_URL): %v", url, err)
+	}
+	_ = resp.Body.Close()
+	return url
+}
+
+// failoverCtx returns a context that forces failover on (the mock is not a
+// cloud host) with a tiny backoff, carrying the given X-Lk-Mock-* directives as
+// twirp request headers.
+func failoverCtx(t *testing.T, mode lksdk.FailoverMode, directives map[string]string) context.Context {
+	ctx := lksdk.WithFailoverConfig(context.Background(), lksdk.FailoverConfig{
+		Mode:        mode,
+		BackoffBase: time.Millisecond,
+	})
+	if len(directives) > 0 {
+		h := make(http.Header)
+		for k, v := range directives {
+			h.Set(k, v)
+		}
+		var err error
+		ctx, err = twirp.WithHTTPRequestHeaders(ctx, h)
+		require.NoError(t, err)
+	}
+	return ctx
+}
+
+func TestAPI_Healthy(t *testing.T) {
+	client := lksdk.NewRoomServiceClient(testServerURL(t), "devkey", "secret")
+	room, err := client.CreateRoom(failoverCtx(t, lksdk.FailoverOn, nil), &livekit.CreateRoomRequest{Name: "api-test"})
+	require.NoError(t, err)
+	require.Equal(t, "api-test", room.Name, "the mock echoes the request name")
+	require.NotEmpty(t, room.Sid)
+}
+
+func TestAPI_PrimaryUnavailable(t *testing.T) {
+	client := lksdk.NewRoomServiceClient(testServerURL(t), "devkey", "secret")
+	ctx := failoverCtx(t, lksdk.FailoverOn, map[string]string{hdrFailRegions: "0"})
+	_, err := client.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: "api-test"})
+	require.NoError(t, err, "should fail over to a healthy region")
+}
+
+func TestAPI_TwoRegionsUnavailable(t *testing.T) {
+	client := lksdk.NewRoomServiceClient(testServerURL(t), "devkey", "secret")
+	ctx := failoverCtx(t, lksdk.FailoverOn, map[string]string{hdrFailRegions: "0,1"})
+	_, err := client.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: "api-test"})
+	require.NoError(t, err, "should fail over to the third region")
+}
+
+func TestAPI_AllUnavailable(t *testing.T) {
+	client := lksdk.NewRoomServiceClient(testServerURL(t), "devkey", "secret")
+	ctx := failoverCtx(t, lksdk.FailoverOn, map[string]string{hdrFailRegions: "0,1,2,3"})
+	_, err := client.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: "api-test"})
+	require.Error(t, err)
+}
+
+func TestAPI_ClientErrorNotRetried(t *testing.T) {
+	client := lksdk.NewRoomServiceClient(testServerURL(t), "devkey", "secret")
+	ctx := failoverCtx(t, lksdk.FailoverOn, map[string]string{hdrFailRegions: "0", hdrFailStatus: "400"})
+	_, err := client.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: "api-test"})
+	require.Error(t, err)
+	var terr twirp.Error
+	require.ErrorAs(t, err, &terr)
+	require.Equal(t, twirp.InvalidArgument, terr.Code(), "a 4xx must surface as a typed error, not fail over")
+}
+
+func TestAPI_TransportError(t *testing.T) {
+	client := lksdk.NewRoomServiceClient(testServerURL(t), "devkey", "secret")
+	ctx := failoverCtx(t, lksdk.FailoverOn, map[string]string{hdrFailRegions: "0", hdrFailMode: "drop"})
+	_, err := client.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: "api-test"})
+	require.NoError(t, err, "a dropped connection should fail over to a healthy region")
+}
+
+func TestAPI_RegionDiscoveryUnreachable(t *testing.T) {
+	client := lksdk.NewRoomServiceClient(testServerURL(t), "devkey", "secret")
+	ctx := failoverCtx(t, lksdk.FailoverOn, map[string]string{hdrFailRegions: "0", hdrRegionsStat: "500"})
+	_, err := client.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: "api-test"})
+	require.Error(t, err, "no fallback hosts means the original error is surfaced")
+}
+
+func TestAPI_FailoverDisabledForNonCloud(t *testing.T) {
+	client := lksdk.NewRoomServiceClient(testServerURL(t), "devkey", "secret")
+	// Auto mode against 127.0.0.1 (non-cloud) must not fail over.
+	ctx := failoverCtx(t, lksdk.FailoverAuto, map[string]string{hdrFailRegions: "0"})
+	_, err := client.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: "api-test"})
+	require.Error(t, err)
+}

connectorclient.go

@@ -16,7 +16,6 @@ package lksdk
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/utils/xtwirp"
@@ -32,7 +31,7 @@ type ConnectorClient struct {
 func NewConnectorClient(url string, apiKey string, secretKey string, opts ...twirp.ClientOption) *ConnectorClient {
 	opts = append(opts, xtwirp.DefaultClientOptions()...)
 	url = signalling.ToHttpURL(url)
-	client := livekit.NewConnectorProtobufClient(url, &http.Client{}, opts...)
+	client := livekit.NewConnectorProtobufClient(url, newAPIHTTPClient(), opts...)
 	return &ConnectorClient{
 		connector: client,
 		authBase: authBase{

egressclient.go

@@ -16,7 +16,6 @@ package lksdk
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/twitchtv/twirp"
 
@@ -33,7 +32,7 @@ type EgressClient struct {
 func NewEgressClient(url string, apiKey string, secretKey string, opts ...twirp.ClientOption) *EgressClient {
 	opts = append(opts, xtwirp.DefaultClientOptions()...)
 	url = signalling.ToHttpURL(url)
-	client := livekit.NewEgressProtobufClient(url, &http.Client{}, opts...)
+	client := livekit.NewEgressProtobufClient(url, newAPIHTTPClient(), opts...)
 	return &EgressClient{
 		egressClient: client,
 		authBase: authBase{