From 15181760c77e29ad4a632574fb6f6802ef1310a0 Mon Sep 17 00:00:00 2001 From: Kalin Ovtcharov Date: Fri, 26 Jun 2026 08:54:25 -0700 Subject: [PATCH 1/6] feat(lemonade): retry transient "llama-server failed to start" on model load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AMD Vulkan iGPU intermittently aborts llama-server startup for some models (notably the nomic-embed-text-v2-moe MoE embedder) with `model_load_error: llama-server failed to start`. Lemonade returns the error in one shot (~10s, no internal retry), but the same load succeeds once the GPU/driver state settles — an upstream llama.cpp #16301 / lemonade #612 defect, not a GAIA bug. LemonadeClient.load_model() now does a small, explicit, bounded retry (default 2, override via load_retries=) ONLY for that transient backend- startup fault — corrupt-download and missing-model failures fall through to their existing handling immediately, and an exhausted retry re-raises with context (fail-loud preserved). This makes RAG embeddings, chat, and VLM loads resilient to the flake instead of hard-failing on the first attempt. --- src/gaia/llm/lemonade_client.py | 70 ++++++++++++++++++++ tests/test_lemonade_client.py | 113 ++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) diff --git a/src/gaia/llm/lemonade_client.py b/src/gaia/llm/lemonade_client.py index 434fb0e78..b5fb5e6e4 100644 --- a/src/gaia/llm/lemonade_client.py +++ b/src/gaia/llm/lemonade_client.py @@ -141,6 +141,12 @@ def lemonade_auth_headers(api_key: Optional[str]) -> Dict[str, str]: # Increased for large model downloads and loading (10x increase for streaming stability) DEFAULT_MODEL_LOAD_TIMEOUT = 12000 +# Resilience to the transient AMD-Vulkan "llama-server failed to start" fault: +# the same load succeeds on a retry once the GPU/driver state settles. Default +# to a small, explicit, bounded retry (callers can override via load_model()). +DEFAULT_MODEL_LOAD_RETRIES = 2 +MODEL_LOAD_RETRY_BACKOFF = 8 # seconds between retries (lets GPU state settle) + # ========================================================================= # Model Types and Agent Profiles @@ -1268,6 +1274,17 @@ def _is_model_error(self, error: Union[str, Dict, Exception]) -> bool: "corrupted download", ) + # Phrases that signal a TRANSIENT backend-startup failure — the same load + # typically succeeds on a retry once the GPU/driver state settles. The + # canonical case is the AMD Vulkan iGPU intermittently aborting + # ``llama-server`` startup for some models (upstream llama.cpp #16301 / + # lemonade #612, not a GAIA defect). Deliberately narrow: a corrupt or + # missing-model failure is NOT transient and must not be retried here. + _TRANSIENT_LOAD_PHRASES = ( + "llama-server failed to start", + "llama_server failed to start", + ) + def _is_corrupt_download_error(self, error: Union[str, Dict, Exception]) -> bool: """ Check if an error indicates a corrupt or incomplete model download. @@ -1288,6 +1305,19 @@ def _is_corrupt_download_error(self, error: Union[str, Dict, Exception]) -> bool return any(phrase in error_message for phrase in self._CORRUPT_DOWNLOAD_PHRASES) + def _is_transient_load_error(self, error: Union[str, Dict, Exception]) -> bool: + """Check whether a load failure is a transient backend-startup fault. + + See :attr:`_TRANSIENT_LOAD_PHRASES`. A corrupt-download failure is + explicitly excluded so the destructive repair path always wins over a + plain retry. + """ + if self._is_corrupt_download_error(error): + return False + error_info = self._extract_error_info(error) + error_message = (error_info.get("message") or "").lower() + return any(phrase in error_message for phrase in self._TRANSIENT_LOAD_PHRASES) + def _execute_with_auto_download( self, api_call: Callable, model: str, auto_download: bool = True ): @@ -2805,6 +2835,7 @@ def load_model( ctx_size: Optional[int] = None, save_options: bool = False, prompt: bool = True, + load_retries: int = DEFAULT_MODEL_LOAD_RETRIES, ) -> Dict[str, Any]: """ Load a model on the server. @@ -2829,6 +2860,12 @@ def load_model( Model will use these settings on future loads. prompt: If True, prompt user before downloading (default: True). Set to False to download automatically without user confirmation. + load_retries: Number of times to retry on a TRANSIENT backend-startup + failure (``llama-server failed to start``) before giving + up. The same load typically succeeds once the GPU/driver + state settles. Default 2; pass 0 to disable. Only the + transient fault is retried — corrupt/missing-model errors + fail through to their normal handling immediately. Returns: Dict containing the status of the load operation @@ -2857,6 +2894,39 @@ def load_model( except Exception as e: original_error = str(e) + # Transient backend-startup fault (e.g. the AMD Vulkan iGPU + # intermittently aborting llama-server): retry the SAME load a + # bounded number of times before any other handling — it usually + # succeeds once the GPU/driver state settles. Corrupt/missing-model + # errors are excluded and fall straight through. + if load_retries > 0 and self._is_transient_load_error(e): + for retry_num in range(1, load_retries + 1): + self.log.warning( + f"{_emoji('⚠️', '[RETRY]')} Transient load failure for " + f"'{model_name}' (retry {retry_num}/{load_retries}): " + f"{original_error}. Backing off " + f"{MODEL_LOAD_RETRY_BACKOFF}s for the backend to recover..." + ) + time.sleep(MODEL_LOAD_RETRY_BACKOFF) + try: + response = self._send_request( + "post", url, request_data, timeout=timeout + ) + self.log.info( + f"{_emoji('✅', '[OK]')} Loaded {model_name} after " + f"{retry_num} retr{'y' if retry_num == 1 else 'ies'}" + ) + self.model = model_name + return response + except Exception as retry_err: # noqa: BLE001 + e = retry_err + original_error = str(retry_err) + if not self._is_transient_load_error(retry_err): + break + # Retries exhausted (or the error changed nature): fall through + # to the existing handling below with the latest error, which + # re-raises with context. Fail-loudly is preserved. + # Check if this is a corrupt/incomplete download error is_corrupt = self._is_corrupt_download_error(e) if is_corrupt: diff --git a/tests/test_lemonade_client.py b/tests/test_lemonade_client.py index eb14b1266..a2455e6cc 100644 --- a/tests/test_lemonade_client.py +++ b/tests/test_lemonade_client.py @@ -690,6 +690,119 @@ def test_load_model(self): # Restore original log level logger.setLevel(original_level) + # ------------------------------------------------------------------ + # Transient "llama-server failed to start" load retry + # ------------------------------------------------------------------ + + _TRANSIENT_LOAD_BODY = { + "error": { + "code": "model_load_error", + "message": ( + f"Failed to load model '{TEST_MODEL}': llama-server failed to start" + ), + "type": "model_load_error", + } + } + + def test_is_transient_load_error_classification(self): + """Only the backend-startup fault counts as transient.""" + self.assertTrue( + self.client._is_transient_load_error("llama-server failed to start") + ) + # Corrupt-download and missing-model failures are NOT transient. + self.assertFalse( + self.client._is_transient_load_error("download validation failed") + ) + self.assertFalse(self.client._is_transient_load_error("model not found")) + + @responses.activate + def test_load_model_retries_transient_then_succeeds(self): + """A transient backend-startup failure is retried and recovers.""" + logger = logging.getLogger("gaia.llm.lemonade_client") + original_level = logger.level + logger.setLevel(logging.CRITICAL) + try: + success = {"status": "success", "message": f"Loaded model: {TEST_MODEL}"} + # First call fails transiently, the retry succeeds. + responses.add( + responses.POST, + f"{API_BASE}/load", + json=self._TRANSIENT_LOAD_BODY, + status=500, + ) + responses.add(responses.POST, f"{API_BASE}/load", json=success, status=200) + with patch("gaia.llm.lemonade_client.time.sleep") as mock_sleep: + result = self.client.load_model(model_name=TEST_MODEL) + self.assertEqual(result, success) + self.assertEqual(len(responses.calls), 2) # initial + one retry + mock_sleep.assert_called_once() # backed off once before retrying + finally: + logger.setLevel(original_level) + + @responses.activate + def test_load_model_retries_exhausted_raises_loudly(self): + """Persistent transient failure re-raises with context after retries.""" + logger = logging.getLogger("gaia.llm.lemonade_client") + original_level = logger.level + logger.setLevel(logging.CRITICAL) + try: + responses.add( + responses.POST, + f"{API_BASE}/load", + json=self._TRANSIENT_LOAD_BODY, + status=500, + ) + with patch("gaia.llm.lemonade_client.time.sleep"): + with self.assertRaises(LemonadeClientError) as ctx: + self.client.load_model(model_name=TEST_MODEL, load_retries=2) + self.assertIn("llama-server failed to start", str(ctx.exception)) + # 1 initial attempt + 2 retries = 3 load calls. + self.assertEqual(len(responses.calls), 3) + finally: + logger.setLevel(original_level) + + @responses.activate + def test_load_model_no_retry_when_disabled(self): + """load_retries=0 fails immediately without retrying or sleeping.""" + logger = logging.getLogger("gaia.llm.lemonade_client") + original_level = logger.level + logger.setLevel(logging.CRITICAL) + try: + responses.add( + responses.POST, + f"{API_BASE}/load", + json=self._TRANSIENT_LOAD_BODY, + status=500, + ) + with patch("gaia.llm.lemonade_client.time.sleep") as mock_sleep: + with self.assertRaises(LemonadeClientError): + self.client.load_model(model_name=TEST_MODEL, load_retries=0) + self.assertEqual(len(responses.calls), 1) # no retry + mock_sleep.assert_not_called() + finally: + logger.setLevel(original_level) + + @responses.activate + def test_load_model_does_not_retry_non_transient(self): + """A non-transient 500 (e.g. generic) is NOT retried.""" + logger = logging.getLogger("gaia.llm.lemonade_client") + original_level = logger.level + logger.setLevel(logging.CRITICAL) + try: + responses.add( + responses.POST, + f"{API_BASE}/load", + json={"error": "Internal server error"}, + status=500, + ) + with patch("gaia.llm.lemonade_client.time.sleep") as mock_sleep: + with self.assertRaises(LemonadeClientError): + self.client.load_model(model_name=TEST_MODEL, load_retries=2) + self.assertEqual(len(responses.calls), 1) # failed immediately + mock_sleep.assert_not_called() + finally: + logger.setLevel(original_level) + @responses.activate def test_unload_model_scoped_vs_global(self): """unload_model(name) targets only that model; unload_model() stays global. From 4e82c07aae1fa917da380a15aaf3edc46bf0f175 Mon Sep 17 00:00:00 2001 From: Kalin Ovtcharov Date: Fri, 26 Jun 2026 09:46:55 -0700 Subject: [PATCH 2/6] test(rag): load embedder via LemonadeClient (exercise the load retry, not raw REST) --- .github/workflows/test_rag.yml | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test_rag.yml b/.github/workflows/test_rag.yml index edbc99d81..a9bc4853a 100644 --- a/.github/workflows/test_rag.yml +++ b/.github/workflows/test_rag.yml @@ -194,23 +194,15 @@ jobs: Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)" } - # Load embedding model into memory (required in Lemonade v9.x) - Write-Host "`n=== Loading Embedding Model ===" - try { - $loadRequest = @{ - model_name = "nomic-embed-text-v2-moe-GGUF" - } | ConvertTo-Json - - Write-Host "Loading model: nomic-embed-text-v2-moe-GGUF" - $loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" ` - -Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 60 - Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)" - } catch { - Write-Host "[ERROR] Model load failed: $($_.Exception.Message)" - if ($_.ErrorDetails) { - Write-Host "Error details: $($_.ErrorDetails.Message)" - } - throw "Failed to load embedding model" + # Load the embedding model THROUGH GAIA's LemonadeClient so this + # integration test exercises the production load path -- including + # the bounded retry that absorbs the transient AMD-Vulkan + # "llama-server failed to start" fault. A raw REST /load here would + # bypass that resilience and flake on the first transient hit. + Write-Host "`n=== Loading Embedding Model (via gaia.llm.LemonadeClient, with retry) ===" + python -c "from gaia.llm.lemonade_client import LemonadeClient; LemonadeClient().load_model('nomic-embed-text-v2-moe-GGUF', prompt=False); print('[OK] embedding model loaded via LemonadeClient')" + if ($LASTEXITCODE -ne 0) { + throw "Failed to load embedding model via LemonadeClient" } # Wait for llamacpp backend to fully initialize (increased from 10s) From be8d8cde6982ac84207bd88861d1933d09b63651 Mon Sep 17 00:00:00 2001 From: Kalin Ovtcharov Date: Fri, 26 Jun 2026 10:02:18 -0700 Subject: [PATCH 3/6] test(ci): drive Lemonade model pull/load/verify through LemonadeClient, not raw REST CI should validate GAIA's actual interface to Lemonade (LemonadeClient) so the production code path -- including the transient-fault model-load retry -- is what gets exercised. A raw REST /load in the harness bypasses that resilience and flakes on the first transient "llama-server failed to start" hit. Adds tests/ci_lemonade_check.py (pull/load/verify a model via the client) and rewires the embedder/LLM setup in test_rag, test_embeddings, and test_api to use it. The raw-server smoke test (test_lemonade_server) and the C++ path stay raw by design (they validate Lemonade's API directly); server health-readiness polls are left as plain GETs (bootstrapping, before the client can connect). --- .github/workflows/test_api.yml | 51 +++-------- .github/workflows/test_embeddings.yml | 94 +++------------------ .github/workflows/test_rag.yml | 116 ++++---------------------- tests/ci_lemonade_check.py | 85 +++++++++++++++++++ 4 files changed, 124 insertions(+), 222 deletions(-) create mode 100644 tests/ci_lemonade_check.py diff --git a/.github/workflows/test_api.yml b/.github/workflows/test_api.yml index 33a4495eb..766bf233c 100644 --- a/.github/workflows/test_api.yml +++ b/.github/workflows/test_api.yml @@ -115,48 +115,17 @@ jobs: throw "Server failed to start" } - # Pull required model for API tests - Write-Host "`n=== Pulling Required Models ===" - Write-Host "Pulling Qwen3-0.6B-GGUF..." - try { - $body = @{ model_name = "Qwen3-0.6B-GGUF" } | ConvertTo-Json - $response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" ` - -Method POST -ContentType "application/json" -Body $body -TimeoutSec 600 - Write-Host " [OK] Qwen3-0.6B-GGUF pull initiated" - } catch { - Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)" - } - - # Wait for model to be available - Write-Host "Waiting 5 seconds for model files to sync..." - Start-Sleep -Seconds 5 - - # Load the model into Lemonade (required for inference) - Write-Host "`n=== Loading LLM Model ===" - try { - $loadRequest = @{ model_name = "Qwen3-0.6B-GGUF" } | ConvertTo-Json - Write-Host "Loading model: Qwen3-0.6B-GGUF" - $loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" ` - -Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 120 - Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)" - } catch { - Write-Host "[ERROR] Model load failed: $($_.Exception.Message)" - if ($_.ErrorDetails) { - Write-Host "Error details: $($_.ErrorDetails.Message)" + # Pull + load + verify the LLM via GAIA's LemonadeClient so CI + # validates our interface to Lemonade (incl. the model-load retry), + # not raw REST. + Write-Host "`n=== Pull + load + verify LLM via LemonadeClient ===" + uv run python tests/ci_lemonade_check.py --model Qwen3-0.6B-GGUF --chat + if ($LASTEXITCODE -ne 0) { + if ($env:LEMONADE_JOB_ID) { + Write-Host "Server output:" + Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue } - } - - # Wait for llamacpp backend initialization - Write-Host "Waiting 10 seconds for llamacpp backend initialization..." - Start-Sleep -Seconds 10 - - # Verify models - try { - $models = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/models" -Method GET - Write-Host "`n[OK] Available models:" - $models.data | ForEach-Object { Write-Host " - $($_.id)" } - } catch { - Write-Host "[WARN] Could not list models: $($_.Exception.Message)" + throw "LLM setup/verify via LemonadeClient failed" } # Start API server using gaia CLI (runs on default port 8080) diff --git a/.github/workflows/test_embeddings.yml b/.github/workflows/test_embeddings.yml index d1bcc0be8..775a31843 100644 --- a/.github/workflows/test_embeddings.yml +++ b/.github/workflows/test_embeddings.yml @@ -107,91 +107,19 @@ jobs: # v10 -- GGUF weights live in the HuggingFace hub cache -- so it was # a silent no-op and has been removed. - # Pull embedding model (actual model used in tests) - Write-Host "`n=== Pulling Embedding Model ===" - Write-Host "Pulling nomic-embed-text-v2-moe-GGUF..." - try { - $body = @{ model_name = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json - $response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" ` - -Method POST -ContentType "application/json" -Body $body -TimeoutSec 600 - Write-Host " [OK] Model pull initiated" - } catch { - Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)" - } - - # Load embedding model into memory (required in Lemonade v9.x) - Write-Host "`n=== Loading Embedding Model ===" - try { - $loadRequest = @{ - model_name = "nomic-embed-text-v2-moe-GGUF" - } | ConvertTo-Json - - Write-Host "Loading model: nomic-embed-text-v2-moe-GGUF" - $loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" ` - -Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 60 - Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)" - } catch { - Write-Host "[ERROR] Model load failed: $($_.Exception.Message)" - if ($_.ErrorDetails) { - Write-Host "Error details: $($_.ErrorDetails.Message)" - } - throw "Failed to load embedding model" - } - - # Wait for llamacpp backend to fully initialize (increased from 10s) - Write-Host "Waiting 30 seconds for llamacpp backend initialization..." - Start-Sleep -Seconds 30 - - # Verify model is available - try { - $models = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/models" -Method GET - Write-Host "`n[OK] Available models:" - $models.data | ForEach-Object { Write-Host " - $($_.id)" } - } catch { - Write-Host "[WARN] Could not verify model: $($_.Exception.Message)" - } - - # Verify server is still responding before embeddings test - Write-Host "`n=== Verifying Server Health ===" - try { - $health = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/health" -Method GET -TimeoutSec 10 - Write-Host "[OK] Server responding: $($health | ConvertTo-Json -Compress)" - } catch { - Write-Host "[ERROR] Server health check failed: $($_.Exception.Message)" - # Show server job output for debugging + # Pull + load + verify the embedding model THROUGH GAIA's + # LemonadeClient so CI validates our actual interface to Lemonade + # (including the model-load retry that absorbs the transient + # AMD-Vulkan "llama-server failed to start" fault) rather than raw + # REST, which would bypass that resilience and flake. + Write-Host "`n=== Pull + load + verify embedding model via LemonadeClient ===" + python tests/ci_lemonade_check.py --model nomic-embed-text-v2-moe-GGUF --embeddings + if ($LASTEXITCODE -ne 0) { if ($env:LEMONADE_JOB_ID) { - $jobOutput = Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue - Write-Host "Server output: $jobOutput" + Write-Host "Server output:" + Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue } - throw "Server not responding after model load" - } - - # Verify embedding model with actual API call - Write-Host "`n=== Verifying Embedding Model ===" - $maxRetries = 3 - $retryCount = 0 - $modelReady = $false - - while ($retryCount -lt $maxRetries -and -not $modelReady) { - try { - $testBody = @{ input = @("test embedding"); model = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json - # Use localhost consistently and increased timeout for first embedding request - $response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/embeddings" ` - -Method POST -ContentType "application/json" -Body $testBody -TimeoutSec 300 - Write-Host "[OK] Embedding model verified successfully" - $modelReady = $true - } catch { - $retryCount++ - Write-Host "[WARN] Embedding verification attempt $retryCount failed: $($_.Exception.Message)" - if ($retryCount -lt $maxRetries) { - Write-Host "Waiting 30 seconds before retry..." - Start-Sleep -Seconds 30 - } - } - } - - if (-not $modelReady) { - throw "Embedding model failed to load after $maxRetries attempts" + throw "Embedding model setup/verify via LemonadeClient failed" } # Run tests in same session while server is running diff --git a/.github/workflows/test_rag.yml b/.github/workflows/test_rag.yml index a9bc4853a..a09e78678 100644 --- a/.github/workflows/test_rag.yml +++ b/.github/workflows/test_rag.yml @@ -158,107 +158,27 @@ jobs: } } - # Pull required models (actual models used in RAG tests) - Write-Host "`n=== Pulling Required Models ===" - - # Pull LLM model - Write-Host "Pulling Qwen3-4B-Instruct-2507-GGUF..." - try { - $body = @{ model_name = "Qwen3-4B-Instruct-2507-GGUF" } | ConvertTo-Json - $response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" ` - -Method POST -ContentType "application/json" -Body $body -TimeoutSec 600 - Write-Host " [OK] Qwen3-4B-Instruct-2507-GGUF pull initiated" - } catch { - Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)" - } - - # Pull embedding model (actual RAG default) - Write-Host "Pulling nomic-embed-text-v2-moe-GGUF..." - try { - $body = @{ model_name = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json - $response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" ` - -Method POST -ContentType "application/json" -Body $body -TimeoutSec 600 - Write-Host " [OK] nomic-embed-text-v2-moe-GGUF pull initiated" - } catch { - Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)" - } - - # Pull VLM model (for PDFs with images) - Write-Host "Pulling Qwen3-VL-4B-Instruct-GGUF..." - try { - $body = @{ model_name = "Qwen3-VL-4B-Instruct-GGUF" } | ConvertTo-Json - $response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" ` - -Method POST -ContentType "application/json" -Body $body -TimeoutSec 1200 - Write-Host " [OK] Qwen3-VL-4B-Instruct-GGUF pull initiated" - } catch { - Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)" - } - - # Load the embedding model THROUGH GAIA's LemonadeClient so this - # integration test exercises the production load path -- including - # the bounded retry that absorbs the transient AMD-Vulkan - # "llama-server failed to start" fault. A raw REST /load here would - # bypass that resilience and flake on the first transient hit. - Write-Host "`n=== Loading Embedding Model (via gaia.llm.LemonadeClient, with retry) ===" - python -c "from gaia.llm.lemonade_client import LemonadeClient; LemonadeClient().load_model('nomic-embed-text-v2-moe-GGUF', prompt=False); print('[OK] embedding model loaded via LemonadeClient')" + # Pull the LLM + VLM via GAIA's LemonadeClient (the embedder is + # pulled+loaded below). CI should drive Lemonade through our client, + # not raw REST. + Write-Host "`n=== Pulling LLM + VLM via LemonadeClient ===" + python tests/ci_lemonade_check.py --model Qwen3-4B-Instruct-2507-GGUF --pull-only + if ($LASTEXITCODE -ne 0) { throw "Failed to pull Qwen3-4B-Instruct-2507-GGUF via LemonadeClient" } + python tests/ci_lemonade_check.py --model Qwen3-VL-4B-Instruct-GGUF --pull-only + if ($LASTEXITCODE -ne 0) { throw "Failed to pull Qwen3-VL-4B-Instruct-GGUF via LemonadeClient" } + + # Load + verify the embedding model THROUGH GAIA's LemonadeClient so + # CI exercises the production load path -- including the bounded + # retry that absorbs the transient AMD-Vulkan "llama-server failed + # to start" fault. Raw REST here would bypass that resilience. + Write-Host "`n=== Load + verify embedding model via LemonadeClient ===" + python tests/ci_lemonade_check.py --model nomic-embed-text-v2-moe-GGUF --embeddings if ($LASTEXITCODE -ne 0) { - throw "Failed to load embedding model via LemonadeClient" - } - - # Wait for llamacpp backend to fully initialize (increased from 10s) - Write-Host "Waiting 30 seconds for llamacpp backend initialization..." - Start-Sleep -Seconds 30 - - # Verify models - try { - $models = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/models" -Method GET - Write-Host "`n[OK] Available models:" - $models.data | ForEach-Object { Write-Host " - $($_.id)" } - } catch { - Write-Host "[WARN] Could not list models: $($_.Exception.Message)" - } - - # Verify server is still responding before embeddings test - Write-Host "`n=== Verifying Server Health ===" - try { - $health = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/health" -Method GET -TimeoutSec 10 - Write-Host "[OK] Server responding: $($health | ConvertTo-Json -Compress)" - } catch { - Write-Host "[ERROR] Server health check failed: $($_.Exception.Message)" - # Show server job output for debugging if ($env:LEMONADE_JOB_ID) { - $jobOutput = Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue - Write-Host "Server output: $jobOutput" - } - throw "Server not responding after model load" - } - - # Verify embedding model with actual API call - Write-Host "`n=== Verifying Embedding Model ===" - $maxRetries = 3 - $retryCount = 0 - $modelReady = $false - - while ($retryCount -lt $maxRetries -and -not $modelReady) { - try { - $testBody = @{ input = @("test embedding"); model = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json - # Use localhost consistently and increased timeout for first embedding request - $response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/embeddings" ` - -Method POST -ContentType "application/json" -Body $testBody -TimeoutSec 300 - Write-Host "[OK] Embedding model verified successfully" - $modelReady = $true - } catch { - $retryCount++ - Write-Host "[WARN] Embedding verification attempt $retryCount failed: $($_.Exception.Message)" - if ($retryCount -lt $maxRetries) { - Write-Host "Waiting 30 seconds before retry..." - Start-Sleep -Seconds 30 - } + Write-Host "Server output:" + Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue } - } - - if (-not $modelReady) { - throw "Embedding model failed to load after $maxRetries attempts" + throw "Embedding model setup/verify via LemonadeClient failed" } # Run tests in same session while server is running diff --git a/tests/ci_lemonade_check.py b/tests/ci_lemonade_check.py new file mode 100644 index 000000000..6f1c25802 --- /dev/null +++ b/tests/ci_lemonade_check.py @@ -0,0 +1,85 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""CI helper: drive a Lemonade model through GAIA's ``LemonadeClient``. + +CI must validate **our interface to Lemonade** (``LemonadeClient``), not raw +REST - so a flaky/transient backend fault is handled exactly as it is in +production (including the model-load retry), and the client surface is what gets +exercised. This pulls + loads a model via the client and optionally verifies an +embeddings or chat round-trip, then exits non-zero on failure. + +Usage: + python tests/ci_lemonade_check.py --model nomic-embed-text-v2-moe-GGUF --embeddings + python tests/ci_lemonade_check.py --model Gemma-4-E4B-it-GGUF --chat --ctx-size 4096 +""" + +import argparse +import sys + +from gaia.llm.lemonade_client import LemonadeClient + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True, help="Lemonade model name") + parser.add_argument( + "--embeddings", action="store_true", help="verify an embeddings round-trip" + ) + parser.add_argument( + "--chat", action="store_true", help="verify a chat-completion round-trip" + ) + parser.add_argument("--ctx-size", type=int, default=None, dest="ctx_size") + parser.add_argument( + "--pull-only", + action="store_true", + dest="pull_only", + help="download the model via the client without loading it", + ) + args = parser.parse_args() + + client = LemonadeClient() + + if args.pull_only: + print("[ci] pull %s via LemonadeClient..." % args.model, flush=True) + client.pull_model(args.model) + print("[ci] pulled %s" % args.model, flush=True) + return 0 + + print("[ci] load %s via LemonadeClient (with retry)..." % args.model, flush=True) + client.load_model( + args.model, prompt=False, auto_download=True, ctx_size=args.ctx_size + ) + print("[ci] loaded %s" % args.model, flush=True) + + # Confirm the model shows up through the client's models surface too. + models = client.list_models() + ids = [m.get("id") for m in (models.get("data") or [])] + print("[ci] models via client: %s" % ", ".join(str(i) for i in ids), flush=True) + + if args.embeddings: + resp = client.embeddings(["ci validation text"], model=args.model) + dim = len(resp["data"][0]["embedding"]) + if dim <= 0: + print("[ci] ERROR: empty embedding vector", flush=True) + return 1 + print("[ci] embeddings OK (dim=%d)" % dim, flush=True) + + if args.chat: + resp = client.chat_completions( + messages=[{"role": "user", "content": "Reply with the word OK."}], + model=args.model, + max_tokens=8, + stream=False, + ) + text = resp["choices"][0]["message"]["content"] + if not text: + print("[ci] ERROR: empty chat completion", flush=True) + return 1 + print("[ci] chat OK (%r)" % text[:40], flush=True) + + print("[ci] OK", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From cab10c0cca9d84a57f1ddaa582846103a1916dc8 Mon Sep 17 00:00:00 2001 From: Kalin Ovtcharov Date: Fri, 26 Jun 2026 14:34:22 -0700 Subject: [PATCH 4/6] feat(lemonade): bump load retry to 3 with escalating backoff (windowed fault) The transient AMD-Vulkan load fault is windowed (consecutive failures for tens of seconds, then clears). Measured: when dormant, 133/133 cold loads succeed; during an active window, attempts fail consecutively past 2x8s. Bump to 3 retries with escalating backoff (8s/16s/24s, ~48s) so a short window is absorbed; a long active window still fails loudly (upstream fix needed). --- src/gaia/llm/lemonade_client.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/gaia/llm/lemonade_client.py b/src/gaia/llm/lemonade_client.py index b39b6564b..c51651ec9 100644 --- a/src/gaia/llm/lemonade_client.py +++ b/src/gaia/llm/lemonade_client.py @@ -142,10 +142,15 @@ def lemonade_auth_headers(api_key: Optional[str]) -> Dict[str, str]: DEFAULT_MODEL_LOAD_TIMEOUT = 12000 # Resilience to the transient AMD-Vulkan "llama-server failed to start" fault: -# the same load succeeds on a retry once the GPU/driver state settles. Default -# to a small, explicit, bounded retry (callers can override via load_model()). -DEFAULT_MODEL_LOAD_RETRIES = 2 -MODEL_LOAD_RETRY_BACKOFF = 8 # seconds between retries (lets GPU state settle) +# the same load succeeds on a retry once the GPU/driver state settles. The fault +# is "windowed" (a bad period of consecutive failures that then clears), so the +# retry uses an ESCALATING backoff to give a short window time to pass. Bounded +# and explicit (callers can override via load_model(load_retries=)). With 3 +# retries the backoff is 8s, 16s, 24s (~48s total) before failing loudly -- a +# one-time model load can afford that; a longer active window needs an upstream +# fix, not unbounded waiting. +DEFAULT_MODEL_LOAD_RETRIES = 3 +MODEL_LOAD_RETRY_BACKOFF = 8 # base seconds; escalates as backoff * attempt # ========================================================================= @@ -2866,7 +2871,8 @@ def load_model( load_retries: Number of times to retry on a TRANSIENT backend-startup failure (``llama-server failed to start``) before giving up. The same load typically succeeds once the GPU/driver - state settles. Default 2; pass 0 to disable. Only the + state settles, with an escalating backoff (8s, 16s, + 24s...). Default 3; pass 0 to disable. Only the transient fault is retried — corrupt/missing-model errors fail through to their normal handling immediately. @@ -2904,13 +2910,14 @@ def load_model( # errors are excluded and fall straight through. if load_retries > 0 and self._is_transient_load_error(e): for retry_num in range(1, load_retries + 1): + backoff = MODEL_LOAD_RETRY_BACKOFF * retry_num self.log.warning( f"{_emoji('⚠️', '[RETRY]')} Transient load failure for " f"'{model_name}' (retry {retry_num}/{load_retries}): " f"{original_error}. Backing off " - f"{MODEL_LOAD_RETRY_BACKOFF}s for the backend to recover..." + f"{backoff}s for the backend to recover..." ) - time.sleep(MODEL_LOAD_RETRY_BACKOFF) + time.sleep(backoff) try: response = self._send_request( "post", url, request_data, timeout=timeout From 9956ad24660a90654cc869330212d08e04df7308 Mon Sep 17 00:00:00 2001 From: Kalin Ovtcharov Date: Fri, 26 Jun 2026 14:48:46 -0700 Subject: [PATCH 5/6] test(rag): tolerate non-fatal LLM/VLM pre-pull failures (match prior behavior) --- .github/workflows/test_rag.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_rag.yml b/.github/workflows/test_rag.yml index a09e78678..029071b54 100644 --- a/.github/workflows/test_rag.yml +++ b/.github/workflows/test_rag.yml @@ -158,14 +158,16 @@ jobs: } } - # Pull the LLM + VLM via GAIA's LemonadeClient (the embedder is - # pulled+loaded below). CI should drive Lemonade through our client, - # not raw REST. - Write-Host "`n=== Pulling LLM + VLM via LemonadeClient ===" + # Pre-pull the LLM + VLM via GAIA's LemonadeClient (the embedder is + # pulled+loaded below). Pulls are a non-fatal pre-warm -- tolerate a + # transient failure and continue (the model auto-downloads on load), + # matching the original behavior. CI drives Lemonade through our + # client, not raw REST. + Write-Host "`n=== Pre-pulling LLM + VLM via LemonadeClient ===" python tests/ci_lemonade_check.py --model Qwen3-4B-Instruct-2507-GGUF --pull-only - if ($LASTEXITCODE -ne 0) { throw "Failed to pull Qwen3-4B-Instruct-2507-GGUF via LemonadeClient" } + if ($LASTEXITCODE -ne 0) { Write-Host " [WARN] LLM pre-pull failed; continuing (auto-downloads on load)" } python tests/ci_lemonade_check.py --model Qwen3-VL-4B-Instruct-GGUF --pull-only - if ($LASTEXITCODE -ne 0) { throw "Failed to pull Qwen3-VL-4B-Instruct-GGUF via LemonadeClient" } + if ($LASTEXITCODE -ne 0) { Write-Host " [WARN] VLM pre-pull failed; continuing (auto-downloads on load)" } # Load + verify the embedding model THROUGH GAIA's LemonadeClient so # CI exercises the production load path -- including the bounded From 59e5c620f99baddd4eb785d0f6dac98799b3cf2a Mon Sep 17 00:00:00 2001 From: Kalin Ovtcharov Date: Fri, 26 Jun 2026 15:08:10 -0700 Subject: [PATCH 6/6] test(ci): tolerate empty content from reasoning models in --chat check Qwen3-0.6B is a reasoning model; with a small max_tokens budget it can spend the whole budget on thinking tokens and return empty content. The chat round-trip through LemonadeClient still succeeded, so assert on a non-empty choices array rather than non-empty content, and raise the token budget to 32. --- tests/ci_lemonade_check.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/ci_lemonade_check.py b/tests/ci_lemonade_check.py index 6f1c25802..0a35c620b 100644 --- a/tests/ci_lemonade_check.py +++ b/tests/ci_lemonade_check.py @@ -68,14 +68,22 @@ def main() -> int: resp = client.chat_completions( messages=[{"role": "user", "content": "Reply with the word OK."}], model=args.model, - max_tokens=8, + max_tokens=32, stream=False, ) - text = resp["choices"][0]["message"]["content"] - if not text: - print("[ci] ERROR: empty chat completion", flush=True) + # Verify the LLM round-trips through the client: a successful response + # with a choices array. Don't require non-empty content -- reasoning + # models can spend the token budget on thinking and return empty + # content, which is not a failure of the round-trip. + choices = resp.get("choices") or [] + if not choices: + print("[ci] ERROR: chat completion returned no choices", flush=True) return 1 - print("[ci] chat OK (%r)" % text[:40], flush=True) + text = (choices[0].get("message") or {}).get("content") or "" + print( + "[ci] chat OK (choices=%d, content=%r)" % (len(choices), text[:40]), + flush=True, + ) print("[ci] OK", flush=True) return 0