Skip to content

feat(lemonade): retry transient "llama-server failed to start" on model load #1876

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kovtcharov wants to merge 7 commits into
base: main
Choose a base branch
from
Open

feat(lemonade): retry transient "llama-server failed to start" on model load #1876

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1518176
feat(lemonade): retry transient "llama-server failed to start" on mod…
kovtcharov Jun 26, 2026
4e82c07
test(rag): load embedder via LemonadeClient (exercise the load retry,…
kovtcharov Jun 26, 2026
282bc1c
Merge remote-tracking branch 'origin/main' into feat/lemonade-load-retry
kovtcharov Jun 26, 2026
be8d8cd
test(ci): drive Lemonade model pull/load/verify through LemonadeClien…
kovtcharov Jun 26, 2026
cab10c0
feat(lemonade): bump load retry to 3 with escalating backoff (windowe…
kovtcharov Jun 26, 2026
9956ad2
test(rag): tolerate non-fatal LLM/VLM pre-pull failures (match prior …
kovtcharov Jun 26, 2026
59e5c62
test(ci): tolerate empty content from reasoning models in --chat check
kovtcharov Jun 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 10 additions & 41 deletions .github/workflows/test_api.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,48 +115,17 @@ jobs:
throw "Server failed to start"
}

# Pull required model for API tests
Write-Host "`n=== Pulling Required Models ==="
Write-Host "Pulling Qwen3-0.6B-GGUF..."
try {
$body = @{ model_name = "Qwen3-0.6B-GGUF" } | ConvertTo-Json
$response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" `
-Method POST -ContentType "application/json" -Body $body -TimeoutSec 600
Write-Host " [OK] Qwen3-0.6B-GGUF pull initiated"
} catch {
Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)"
}

# Wait for model to be available
Write-Host "Waiting 5 seconds for model files to sync..."
Start-Sleep -Seconds 5

# Load the model into Lemonade (required for inference)
Write-Host "`n=== Loading LLM Model ==="
try {
$loadRequest = @{ model_name = "Qwen3-0.6B-GGUF" } | ConvertTo-Json
Write-Host "Loading model: Qwen3-0.6B-GGUF"
$loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" `
-Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 120
Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)"
} catch {
Write-Host "[ERROR] Model load failed: $($_.Exception.Message)"
if ($_.ErrorDetails) {
Write-Host "Error details: $($_.ErrorDetails.Message)"
# Pull + load + verify the LLM via GAIA's LemonadeClient so CI
# validates our interface to Lemonade (incl. the model-load retry),
# not raw REST.
Write-Host "`n=== Pull + load + verify LLM via LemonadeClient ==="
uv run python tests/ci_lemonade_check.py --model Qwen3-0.6B-GGUF --chat
if ($LASTEXITCODE -ne 0) {
if ($env:LEMONADE_JOB_ID) {
Write-Host "Server output:"
Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue
}
}

# Wait for llamacpp backend initialization
Write-Host "Waiting 10 seconds for llamacpp backend initialization..."
Start-Sleep -Seconds 10

# Verify models
try {
$models = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/models" -Method GET
Write-Host "`n[OK] Available models:"
$models.data | ForEach-Object { Write-Host " - $($_.id)" }
} catch {
Write-Host "[WARN] Could not list models: $($_.Exception.Message)"
throw "LLM setup/verify via LemonadeClient failed"
}

# Start API server using gaia CLI (runs on default port 8080)
Expand Down
94 changes: 11 additions & 83 deletions .github/workflows/test_embeddings.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,91 +107,19 @@ jobs:
# v10 -- GGUF weights live in the HuggingFace hub cache -- so it was
# a silent no-op and has been removed.

# Pull embedding model (actual model used in tests)
Write-Host "`n=== Pulling Embedding Model ==="
Write-Host "Pulling nomic-embed-text-v2-moe-GGUF..."
try {
$body = @{ model_name = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json
$response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" `
-Method POST -ContentType "application/json" -Body $body -TimeoutSec 600
Write-Host " [OK] Model pull initiated"
} catch {
Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)"
}

# Load embedding model into memory (required in Lemonade v9.x)
Write-Host "`n=== Loading Embedding Model ==="
try {
$loadRequest = @{
model_name = "nomic-embed-text-v2-moe-GGUF"
} | ConvertTo-Json

Write-Host "Loading model: nomic-embed-text-v2-moe-GGUF"
$loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" `
-Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 60
Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)"
} catch {
Write-Host "[ERROR] Model load failed: $($_.Exception.Message)"
if ($_.ErrorDetails) {
Write-Host "Error details: $($_.ErrorDetails.Message)"
}
throw "Failed to load embedding model"
}

# Wait for llamacpp backend to fully initialize (increased from 10s)
Write-Host "Waiting 30 seconds for llamacpp backend initialization..."
Start-Sleep -Seconds 30

# Verify model is available
try {
$models = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/models" -Method GET
Write-Host "`n[OK] Available models:"
$models.data | ForEach-Object { Write-Host " - $($_.id)" }
} catch {
Write-Host "[WARN] Could not verify model: $($_.Exception.Message)"
}

# Verify server is still responding before embeddings test
Write-Host "`n=== Verifying Server Health ==="
try {
$health = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/health" -Method GET -TimeoutSec 10
Write-Host "[OK] Server responding: $($health | ConvertTo-Json -Compress)"
} catch {
Write-Host "[ERROR] Server health check failed: $($_.Exception.Message)"
# Show server job output for debugging
# Pull + load + verify the embedding model THROUGH GAIA's
# LemonadeClient so CI validates our actual interface to Lemonade
# (including the model-load retry that absorbs the transient
# AMD-Vulkan "llama-server failed to start" fault) rather than raw
# REST, which would bypass that resilience and flake.
Write-Host "`n=== Pull + load + verify embedding model via LemonadeClient ==="
python tests/ci_lemonade_check.py --model nomic-embed-text-v2-moe-GGUF --embeddings
if ($LASTEXITCODE -ne 0) {
if ($env:LEMONADE_JOB_ID) {
$jobOutput = Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue
Write-Host "Server output: $jobOutput"
Write-Host "Server output:"
Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue
}
throw "Server not responding after model load"
}

# Verify embedding model with actual API call
Write-Host "`n=== Verifying Embedding Model ==="
$maxRetries = 3
$retryCount = 0
$modelReady = $false

while ($retryCount -lt $maxRetries -and -not $modelReady) {
try {
$testBody = @{ input = @("test embedding"); model = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json
# Use localhost consistently and increased timeout for first embedding request
$response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/embeddings" `
-Method POST -ContentType "application/json" -Body $testBody -TimeoutSec 300
Write-Host "[OK] Embedding model verified successfully"
$modelReady = $true
} catch {
$retryCount++
Write-Host "[WARN] Embedding verification attempt $retryCount failed: $($_.Exception.Message)"
if ($retryCount -lt $maxRetries) {
Write-Host "Waiting 30 seconds before retry..."
Start-Sleep -Seconds 30
}
}
}

if (-not $modelReady) {
throw "Embedding model failed to load after $maxRetries attempts"
throw "Embedding model setup/verify via LemonadeClient failed"
}

# Run tests in same session while server is running
Expand Down
128 changes: 21 additions & 107 deletions .github/workflows/test_rag.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,115 +158,29 @@ jobs:
}
}

# Pull required models (actual models used in RAG tests)
Write-Host "`n=== Pulling Required Models ==="

# Pull LLM model
Write-Host "Pulling Qwen3-4B-Instruct-2507-GGUF..."
try {
$body = @{ model_name = "Qwen3-4B-Instruct-2507-GGUF" } | ConvertTo-Json
$response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" `
-Method POST -ContentType "application/json" -Body $body -TimeoutSec 600
Write-Host " [OK] Qwen3-4B-Instruct-2507-GGUF pull initiated"
} catch {
Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)"
}

# Pull embedding model (actual RAG default)
Write-Host "Pulling nomic-embed-text-v2-moe-GGUF..."
try {
$body = @{ model_name = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json
$response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" `
-Method POST -ContentType "application/json" -Body $body -TimeoutSec 600
Write-Host " [OK] nomic-embed-text-v2-moe-GGUF pull initiated"
} catch {
Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)"
}

# Pull VLM model (for PDFs with images)
Write-Host "Pulling Qwen3-VL-4B-Instruct-GGUF..."
try {
$body = @{ model_name = "Qwen3-VL-4B-Instruct-GGUF" } | ConvertTo-Json
$response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/pull" `
-Method POST -ContentType "application/json" -Body $body -TimeoutSec 1200
Write-Host " [OK] Qwen3-VL-4B-Instruct-GGUF pull initiated"
} catch {
Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)"
}

# Load embedding model into memory (required in Lemonade v9.x)
Write-Host "`n=== Loading Embedding Model ==="
try {
$loadRequest = @{
model_name = "nomic-embed-text-v2-moe-GGUF"
} | ConvertTo-Json

Write-Host "Loading model: nomic-embed-text-v2-moe-GGUF"
$loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" `
-Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 60
Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)"
} catch {
Write-Host "[ERROR] Model load failed: $($_.Exception.Message)"
if ($_.ErrorDetails) {
Write-Host "Error details: $($_.ErrorDetails.Message)"
}
throw "Failed to load embedding model"
}

# Wait for llamacpp backend to fully initialize (increased from 10s)
Write-Host "Waiting 30 seconds for llamacpp backend initialization..."
Start-Sleep -Seconds 30

# Verify models
try {
$models = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/models" -Method GET
Write-Host "`n[OK] Available models:"
$models.data | ForEach-Object { Write-Host " - $($_.id)" }
} catch {
Write-Host "[WARN] Could not list models: $($_.Exception.Message)"
}

# Verify server is still responding before embeddings test
Write-Host "`n=== Verifying Server Health ==="
try {
$health = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/health" -Method GET -TimeoutSec 10
Write-Host "[OK] Server responding: $($health | ConvertTo-Json -Compress)"
} catch {
Write-Host "[ERROR] Server health check failed: $($_.Exception.Message)"
# Show server job output for debugging
# Pre-pull the LLM + VLM via GAIA's LemonadeClient (the embedder is
# pulled+loaded below). Pulls are a non-fatal pre-warm -- tolerate a
# transient failure and continue (the model auto-downloads on load),
# matching the original behavior. CI drives Lemonade through our
# client, not raw REST.
Write-Host "`n=== Pre-pulling LLM + VLM via LemonadeClient ==="
python tests/ci_lemonade_check.py --model Qwen3-4B-Instruct-2507-GGUF --pull-only
if ($LASTEXITCODE -ne 0) { Write-Host " [WARN] LLM pre-pull failed; continuing (auto-downloads on load)" }
python tests/ci_lemonade_check.py --model Qwen3-VL-4B-Instruct-GGUF --pull-only
if ($LASTEXITCODE -ne 0) { Write-Host " [WARN] VLM pre-pull failed; continuing (auto-downloads on load)" }

# Load + verify the embedding model THROUGH GAIA's LemonadeClient so
# CI exercises the production load path -- including the bounded
# retry that absorbs the transient AMD-Vulkan "llama-server failed
# to start" fault. Raw REST here would bypass that resilience.
Write-Host "`n=== Load + verify embedding model via LemonadeClient ==="
python tests/ci_lemonade_check.py --model nomic-embed-text-v2-moe-GGUF --embeddings
if ($LASTEXITCODE -ne 0) {
if ($env:LEMONADE_JOB_ID) {
$jobOutput = Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue
Write-Host "Server output: $jobOutput"
Write-Host "Server output:"
Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue
}
throw "Server not responding after model load"
}

# Verify embedding model with actual API call
Write-Host "`n=== Verifying Embedding Model ==="
$maxRetries = 3
$retryCount = 0
$modelReady = $false

while ($retryCount -lt $maxRetries -and -not $modelReady) {
try {
$testBody = @{ input = @("test embedding"); model = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json
# Use localhost consistently and increased timeout for first embedding request
$response = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/embeddings" `
-Method POST -ContentType "application/json" -Body $testBody -TimeoutSec 300
Write-Host "[OK] Embedding model verified successfully"
$modelReady = $true
} catch {
$retryCount++
Write-Host "[WARN] Embedding verification attempt $retryCount failed: $($_.Exception.Message)"
if ($retryCount -lt $maxRetries) {
Write-Host "Waiting 30 seconds before retry..."
Start-Sleep -Seconds 30
}
}
}

if (-not $modelReady) {
throw "Embedding model failed to load after $maxRetries attempts"
throw "Embedding model setup/verify via LemonadeClient failed"
}

# Run tests in same session while server is running
Expand Down
Loading
Loading