From e04aa2200201b7325b7cc8460727dda68bc7fec6 Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Sat, 4 Jul 2026 11:49:51 +0200 Subject: [PATCH] fix(remote-signer): keep retrying discovery while the cache is empty The remote-signer /discover-orchestrators cache is derived from the node's network-capabilities snapshot, which the orchestrator pool populates on its first poll (asynchronously at startup). remoteDiscoveryPool.refresh() advanced lastRefresh unconditionally, so a request that landed before that first poll completed locked in an empty snapshot for a full refreshEvery interval (= -liveAICapReportInterval, default 25m). Until then /discover-orchestrators returned "503 cache empty", and the only workaround was setting a very short -liveAICapReportInterval. Rate-limit refreshes only once a non-empty snapshot exists. While the cache is empty, every call re-derives from the in-memory GetNetworkCapabilities() snapshot (no network I/O), so orchestrators surface as soon as they appear instead of after a full interval. Co-Authored-By: Claude Opus 4.8 (1M context) --- server/remote_discovery.go | 8 +++++- server/remote_signer_test.go | 55 ++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/server/remote_discovery.go b/server/remote_discovery.go index bd08efd665..cb89234155 100644 --- a/server/remote_discovery.go +++ b/server/remote_discovery.go @@ -116,7 +116,13 @@ func (p *remoteDiscoveryPool) refresh() { p.mu.Lock() defer p.mu.Unlock() now := time.Now() - if !p.lastRefresh.IsZero() && now.Sub(p.lastRefresh) <= p.refreshEvery { + // Rate-limit refreshes only once we hold a populated snapshot. While the cache + // is empty (e.g. right after startup, before the orchestrator pool's first poll + // has populated the node's network capabilities), keep re-deriving on every call + // so orchestrators surface as soon as they appear instead of being locked out for + // a full refresh interval. refresh() itself does no network I/O — it only reads + // the in-memory GetNetworkCapabilities() snapshot — so retrying while empty is cheap. + if len(p.cached) > 0 && !p.lastRefresh.IsZero() && now.Sub(p.lastRefresh) <= p.refreshEvery { return } diff --git a/server/remote_signer_test.go b/server/remote_signer_test.go index 9e2d7a154a..51680a4967 100644 --- a/server/remote_signer_test.go +++ b/server/remote_signer_test.go @@ -1311,6 +1311,61 @@ func TestRemoteSigner_Discovery(t *testing.T) { require.Equal("application/json", ineligibleRR.Header().Get("Content-Type")) } +// TestRemoteSigner_Discovery_EmptyCacheRetriesBeforeInterval guards against the +// startup race where the first refresh runs before the orchestrator pool's poll has +// populated the node's network capabilities. An empty snapshot must not be locked in +// for a full refresh interval: while the cache is empty, every request re-derives, so +// orchestrators surface as soon as they appear rather than after -liveAICapReportInterval. +func TestRemoteSigner_Discovery_EmptyCacheRetriesBeforeInterval(t *testing.T) { + require := require.New(t) + + capability := core.Capability_LiveVideoToVideo + modelID := "scope" + BroadcastCfg.SetCapabilityMaxPrice(capability, modelID, core.NewFixedPrice(big.NewRat(200, 995328000000))) + defer BroadcastCfg.SetCapabilityMaxPrice(capability, modelID, nil) + + // Node starts with no network capabilities (pool poll has not populated the cache yet). + node := &core.LivepeerNode{} + + // Long interval: with the bug, an empty first refresh would lock discovery out for an hour. + rdp := &remoteDiscoveryPool{ + node: node, + refreshEvery: time.Hour, + } + ls := &LivepeerServer{} + + // First request lands before the cache is populated: 503, cache empty. + emptyReq := httptest.NewRequest(http.MethodGet, "/discover-orchestrators?caps=live-video-to-video/scope", nil) + emptyRR := httptest.NewRecorder() + ls.GetOrchestrators(rdp, emptyRR, emptyReq) + require.Equal(http.StatusServiceUnavailable, emptyRR.Code) + + // The orchestrator pool finishes its poll and populates network capabilities. + require.NoError(node.UpdateNetworkCapabilities([]*common.OrchNetworkCapabilities{ + { + OrchURI: "https://late.example.com:8935", + Discovery: discoveryRaw(t, `[{ + "address": "https://late.example.com:8935", + "runners": [ + {"url":"https://late.example.com:8935/apps/runner/session","app":"live-video-to-video/scope","capacity":1,"price_info":{"price_per_unit":5,"pixels_per_unit":995328000000,"unit":"WEI"}} + ] + }]`), + }, + })) + + // A follow-up request arrives well within refreshEvery. With the fix it re-derives + // (empty cache is not rate-limited) and returns the newly-registered orchestrator. + req := httptest.NewRequest(http.MethodGet, "/discover-orchestrators?caps=live-video-to-video/scope", nil) + rr := httptest.NewRecorder() + ls.GetOrchestrators(rdp, rr, req) + + require.Equal(http.StatusOK, rr.Code) + var resp []discoveryResponse + require.NoError(json.NewDecoder(rr.Body).Decode(&resp)) + require.Len(resp, 1) + require.Equal("https://late.example.com:8935", resp[0].Address) +} + func TestRemoteSigner_Discovery_UsesRunnerDiscoveryWhenRPCCapabilitiesMissing(t *testing.T) { require := require.New(t)