From 53b7c574983b1bde71deb0e00f9e9efa46e736e0 Mon Sep 17 00:00:00 2001
From: nataliasocaity
Date: Thu, 4 Jun 2026 13:53:38 +0100
Subject: [PATCH 01/38] schemas: proposal for the 6 missing Pydantic schemas +
FRICTION #15 (#179)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Following the research summary at tareas/research_225/_proposal.html, this is a draft
implementation of the six modality schemas for the APIPod common layer. Pushing as
a proposal for review — happy to throw it away or rewrite.
What it adds:
- 12 Pydantic classes in apipod/common/schemas.py (Request + Response for image
generation, video generation, audio, 3D, vision, multimodal embedding).
- arbitrary_types_allowed on OpenAIBaseModel so media_toolkit's ImageFile /
AudioFile / VideoFile can be used as field annotations. APIPod's
file_handling_mixin converts URL / base64 inputs into those types before
validation; the schemas don't need to do their own conversion.
- 6 entries in _BaseLLMMixin._llm_configs mapping each new request to (response,
endpoint_type). Endpoint type strings: image_generation, video_generation,
audio, generation_3d, vision, embedding_multimodal.
- 6 elif branches in _wrap_llm_response that build the right response object
per endpoint_type. The chat / completion / embedding branches are the template.
Two bugs in the existing auto-wrap path fixed on the same pass — without these
the wrap never reaches the new branches:
- FRICTION #15 (line 87): wrap was passing object='embedding' to
EmbeddingResponse, but the schema declares object: Literal['list']. This is
why qwen-models's /embed endpoint builds the response by hand. After the fix
the auto-wrap path is safe; qwen-models can drop its manual EmbeddingResponse
construction once this lands.
- Line 81: int(datetime.now(timezone.utc)) raises TypeError on every call
(datetime is not int-castable). Replaced with int(...timestamp()). The wrap
path was unreachable before this fix, so I can't tell from git history when
it broke — flagging in case it's intentional.
Tests in test/llm/test_six_schemas.py (21 tests, all passing). No torch /
transformers required. Covers:
- Registry shape (9 entries in the tuple and in _llm_configs).
- Required-field validation per new schema.
- Documented defaults per new schema.
- Response Literal['...'] validation per new modality.
- Full wrap path for each new endpoint_type.
- FRICTION #15 regression (outer object == 'list', data[i].object == 'embedding').
- Unknown endpoint_type still raises ValueError.
What's out of scope for this PR (deliberate):
- Generalising _BaseLLMMixin to _BaseModalityMixin (these new forms aren't LLM;
the mixin name doesn't quite fit anymore). Architectural call.
- Consolidating SUPPORTED_LLM_REQUEST_SCHEMAS (in schemas.py) and _llm_configs
(in the mixin) into a single source of truth. Cleanup PR.
- NSFW (#164) wiring — its own backlog item.
- Wiring any of these schemas to a real model service.
Questions for review:
1. Field set per schema — yes / change-this-field / split-into-two?
2. Class names — happy to swap.
3. The two existing bugs (FRICTION #15 + line 81 datetime). Want them in this
PR or split out?
Refs: tareas/research_225/_proposal.html
---
apipod/common/schemas.py | 219 ++++++++++++++++++++++++
apipod/engine/llm/base_llm_mixin.py | 85 +++++++++-
test/llm/test_six_schemas.py | 255 ++++++++++++++++++++++++++++
3 files changed, 553 insertions(+), 6 deletions(-)
create mode 100644 test/llm/test_six_schemas.py
diff --git a/apipod/common/schemas.py b/apipod/common/schemas.py
index 488918e..89e664a 100644
--- a/apipod/common/schemas.py
+++ b/apipod/common/schemas.py
@@ -1,6 +1,8 @@
from typing import List, Optional, Union, Literal
from pydantic import BaseModel
+from media_toolkit import ImageFile, AudioFile
+
# =====================================================
# Base schema
# =====================================================
@@ -10,6 +12,10 @@ class OpenAIBaseModel(BaseModel):
"extra": "forbid",
"validate_assignment": True,
"populate_by_name": True,
+ # Allow media_toolkit file types (ImageFile, AudioFile, VideoFile, MediaFile)
+ # as field annotations. APIPod's file_handling_mixin converts URL/base64
+ # inputs into these types before request validation.
+ "arbitrary_types_allowed": True,
}
@@ -63,12 +69,104 @@ class EmbeddingRequest(OpenAIBaseModel):
user: Optional[str] = None
+# =====================================================
+# Image Generation - Input schemas
+# =====================================================
+
+class ImageGenerationRequest(OpenAIBaseModel):
+ model: str
+ prompt: str
+
+ negative_prompt: Optional[str] = None
+ image: Optional[ImageFile] = None
+ mask: Optional[ImageFile] = None
+ size: Optional[str] = None
+ num_images: int = 1
+ seed: Optional[int] = None
+ steps: Optional[int] = None
+
+
+# =====================================================
+# Video Generation - Input schemas
+# =====================================================
+
+class VideoGenerationRequest(OpenAIBaseModel):
+ model: str
+ prompt: str
+
+ image: Optional[ImageFile] = None
+ duration_s: float = 5.0
+ fps: int = 24
+ aspect_ratio: Optional[str] = None
+ seed: Optional[int] = None
+
+
+# =====================================================
+# Audio - Input schemas (TTS, STT, music)
+# =====================================================
+
+class AudioRequest(OpenAIBaseModel):
+ model: str
+
+ text: Optional[str] = None
+ audio: Optional[AudioFile] = None
+ voice: Optional[str] = None
+ language: Optional[str] = None
+ format: Optional[str] = None
+ duration_s: Optional[float] = None
+
+
+# =====================================================
+# 3D Generation - Input schemas
+# =====================================================
+
+class Generation3DRequest(OpenAIBaseModel):
+ model: str
+
+ prompt: Optional[str] = None
+ image: Optional[ImageFile] = None
+ output_format: str = "glb"
+ seed: Optional[int] = None
+
+
+# =====================================================
+# Vision - Input schemas (classify, detect, OCR)
+# =====================================================
+
+class VisionRequest(OpenAIBaseModel):
+ model: str
+ image: ImageFile
+
+ labels: Optional[List[str]] = None
+ threshold: Optional[float] = None
+ return_boxes: bool = False
+
+
+# =====================================================
+# Multimodal Embedding - Input schemas
+# =====================================================
+
+class MultimodalEmbeddingRequest(OpenAIBaseModel):
+ model: str
+
+ input: Optional[Union[str, List[str]]] = None
+ image: Optional[ImageFile] = None
+ audio: Optional[AudioFile] = None
+ user: Optional[str] = None
+
+
# Supported request schemas that should be interpreted as JSON bodies
# by router decorators, even when endpoint authors do not specify Body(...).
SUPPORTED_LLM_REQUEST_SCHEMAS = (
ChatCompletionRequest,
CompletionRequest,
EmbeddingRequest,
+ ImageGenerationRequest,
+ VideoGenerationRequest,
+ AudioRequest,
+ Generation3DRequest,
+ VisionRequest,
+ MultimodalEmbeddingRequest,
)
@@ -142,6 +240,127 @@ class EmbeddingResponse(OpenAIBaseModel):
model: str
usage: Usage
+
+# =====================================================
+# Image Generation - Output schemas
+# =====================================================
+
+class ImageGenerationData(OpenAIBaseModel):
+ url: Optional[str] = None
+ b64_json: Optional[str] = None
+ revised_prompt: Optional[str] = None
+ seed: Optional[int] = None
+
+
+class ImageGenerationResponse(OpenAIBaseModel):
+ id: str
+ object: Literal["image_generation"]
+ created: int
+ model: str
+ data: List[ImageGenerationData]
+ usage: Optional[Usage] = None
+
+
+# =====================================================
+# Video Generation - Output schemas
+# =====================================================
+
+class VideoGenerationData(OpenAIBaseModel):
+ url: Optional[str] = None
+ duration_s: Optional[float] = None
+ seed: Optional[int] = None
+
+
+class VideoGenerationResponse(OpenAIBaseModel):
+ id: str
+ object: Literal["video_generation"]
+ created: int
+ model: str
+ data: List[VideoGenerationData]
+ usage: Optional[Usage] = None
+
+
+# =====================================================
+# Audio - Output schemas
+# =====================================================
+
+class AudioData(OpenAIBaseModel):
+ audio: Optional[str] = None
+ text: Optional[str] = None
+ language: Optional[str] = None
+ duration_s: Optional[float] = None
+
+
+class AudioResponse(OpenAIBaseModel):
+ id: str
+ object: Literal["audio"]
+ created: int
+ model: str
+ data: List[AudioData]
+ usage: Optional[Usage] = None
+
+
+# =====================================================
+# 3D Generation - Output schemas
+# =====================================================
+
+class Generation3DData(OpenAIBaseModel):
+ url: Optional[str] = None
+ output_format: Optional[str] = None
+ seed: Optional[int] = None
+
+
+class Generation3DResponse(OpenAIBaseModel):
+ id: str
+ object: Literal["generation_3d"]
+ created: int
+ model: str
+ data: List[Generation3DData]
+ usage: Optional[Usage] = None
+
+
+# =====================================================
+# Vision - Output schemas (classify, detect, OCR)
+# =====================================================
+
+class VisionLabel(OpenAIBaseModel):
+ label: str
+ score: float
+ box: Optional[List[float]] = None
+
+
+class VisionData(OpenAIBaseModel):
+ labels: List[VisionLabel] = []
+ text: Optional[str] = None
+
+
+class VisionResponse(OpenAIBaseModel):
+ id: str
+ object: Literal["vision"]
+ created: int
+ model: str
+ data: List[VisionData]
+ usage: Optional[Usage] = None
+
+
+# =====================================================
+# Multimodal Embedding - Output schemas
+# =====================================================
+
+class MultimodalEmbeddingData(OpenAIBaseModel):
+ object: Literal["embedding"] = "embedding"
+ embedding: List[float]
+ index: int
+ modality: Optional[Literal["text", "image", "audio"]] = None
+
+
+class MultimodalEmbeddingResponse(OpenAIBaseModel):
+ object: Literal["list"]
+ data: List[MultimodalEmbeddingData]
+ model: str
+ usage: Optional[Usage] = None
+
+
# ======================================================
# Streaming Models
# ======================================================
diff --git a/apipod/engine/llm/base_llm_mixin.py b/apipod/engine/llm/base_llm_mixin.py
index 674efff..e27cd63 100644
--- a/apipod/engine/llm/base_llm_mixin.py
+++ b/apipod/engine/llm/base_llm_mixin.py
@@ -3,7 +3,17 @@
import uuid
from datetime import datetime, timezone
import inspect
-from apipod.common.schemas import ChatCompletionRequest, ChatCompletionResponse, CompletionRequest, CompletionResponse, EmbeddingRequest, EmbeddingResponse, ChatCompletionChoice, CompletionChoice, EmbeddingData
+from apipod.common.schemas import (
+ ChatCompletionRequest, ChatCompletionResponse, ChatCompletionChoice,
+ CompletionRequest, CompletionResponse, CompletionChoice,
+ EmbeddingRequest, EmbeddingResponse, EmbeddingData,
+ ImageGenerationRequest, ImageGenerationResponse, ImageGenerationData,
+ VideoGenerationRequest, VideoGenerationResponse, VideoGenerationData,
+ AudioRequest, AudioResponse, AudioData,
+ Generation3DRequest, Generation3DResponse, Generation3DData,
+ VisionRequest, VisionResponse, VisionData,
+ MultimodalEmbeddingRequest, MultimodalEmbeddingResponse, MultimodalEmbeddingData,
+)
class _BaseLLMMixin:
@@ -13,9 +23,15 @@ class _BaseLLMMixin:
def __init__(self, *args, **kwargs):
self._llm_configs = {
- ChatCompletionRequest: (ChatCompletionResponse, "chat"),
- CompletionRequest: (CompletionResponse, "completion"),
- EmbeddingRequest: (EmbeddingResponse, "embedding"),
+ ChatCompletionRequest: (ChatCompletionResponse, "chat"),
+ CompletionRequest: (CompletionResponse, "completion"),
+ EmbeddingRequest: (EmbeddingResponse, "embedding"),
+ ImageGenerationRequest: (ImageGenerationResponse, "image_generation"),
+ VideoGenerationRequest: (VideoGenerationResponse, "video_generation"),
+ AudioRequest: (AudioResponse, "audio"),
+ Generation3DRequest: (Generation3DResponse, "generation_3d"),
+ VisionRequest: (VisionResponse, "vision"),
+ MultimodalEmbeddingRequest: (MultimodalEmbeddingResponse, "embedding_multimodal"),
}
self._supported_llm_request_models = tuple(self._llm_configs.keys())
@@ -62,7 +78,7 @@ def _wrap_llm_response(self, result: Any, response_model: Type, endpoint_type: s
return result
model_name = getattr(openai_req, "model", "unknown-model")
- ts, uid = int(datetime.now(timezone.utc)), uuid.uuid4().hex[:8]
+ ts, uid = int(datetime.now(timezone.utc).timestamp()), uuid.uuid4().hex[:8]
if endpoint_type == "chat":
return response_model(
@@ -83,11 +99,68 @@ def _wrap_llm_response(self, result: Any, response_model: Type, endpoint_type: s
usage=result.get("usage")
)
elif endpoint_type == "embedding":
+ # FRICTION #15 fix: the outer object on EmbeddingResponse is "list"
+ # (each item in data carries object="embedding"). The schema declares
+ # object: Literal["list"]; passing "embedding" here used to fail
+ # Pydantic validation, which is why qwen-models constructs the
+ # response by hand. After this fix, the auto-wrap path is safe.
return response_model(
- object="embedding",
+ object="list",
data=[EmbeddingData(**data) for data in result["data"]],
model=model_name,
usage=result.get("usage")
)
+ elif endpoint_type == "image_generation":
+ return response_model(
+ id=f"imggen-{ts}-{uid}",
+ object="image_generation",
+ created=ts,
+ model=model_name,
+ data=[ImageGenerationData(**item) for item in result["data"]],
+ usage=result.get("usage")
+ )
+ elif endpoint_type == "video_generation":
+ return response_model(
+ id=f"vidgen-{ts}-{uid}",
+ object="video_generation",
+ created=ts,
+ model=model_name,
+ data=[VideoGenerationData(**item) for item in result["data"]],
+ usage=result.get("usage")
+ )
+ elif endpoint_type == "audio":
+ return response_model(
+ id=f"aud-{ts}-{uid}",
+ object="audio",
+ created=ts,
+ model=model_name,
+ data=[AudioData(**item) for item in result["data"]],
+ usage=result.get("usage")
+ )
+ elif endpoint_type == "generation_3d":
+ return response_model(
+ id=f"gen3d-{ts}-{uid}",
+ object="generation_3d",
+ created=ts,
+ model=model_name,
+ data=[Generation3DData(**item) for item in result["data"]],
+ usage=result.get("usage")
+ )
+ elif endpoint_type == "vision":
+ return response_model(
+ id=f"vis-{ts}-{uid}",
+ object="vision",
+ created=ts,
+ model=model_name,
+ data=[VisionData(**item) for item in result["data"]],
+ usage=result.get("usage")
+ )
+ elif endpoint_type == "embedding_multimodal":
+ return response_model(
+ object="list",
+ data=[MultimodalEmbeddingData(**item) for item in result["data"]],
+ model=model_name,
+ usage=result.get("usage")
+ )
else:
raise ValueError(f"Unknown endpoint type: {endpoint_type}")
\ No newline at end of file
diff --git a/test/llm/test_six_schemas.py b/test/llm/test_six_schemas.py
new file mode 100644
index 0000000..d4f7a06
--- /dev/null
+++ b/test/llm/test_six_schemas.py
@@ -0,0 +1,255 @@
+"""
+Tests for the six modality schemas added in #179, plus the FRICTION #15 fix
+in _BaseLLMMixin._wrap_llm_response.
+
+These tests do not require torch / transformers — they exercise the schema
+validation path and the mixin wrap path directly.
+"""
+
+import pytest
+from pydantic import ValidationError
+
+from apipod.common import schemas
+from apipod.engine.llm.base_llm_mixin import _BaseLLMMixin
+
+
+# ----------------------------------------------------------------------
+# Registry / supported tuple
+# ----------------------------------------------------------------------
+
+def test_supported_tuple_has_nine_entries():
+ """SUPPORTED_LLM_REQUEST_SCHEMAS should now hold 3 original + 6 new = 9."""
+ assert len(schemas.SUPPORTED_LLM_REQUEST_SCHEMAS) == 9
+
+
+def test_supported_tuple_contains_new_request_classes():
+ for cls in (
+ schemas.ImageGenerationRequest,
+ schemas.VideoGenerationRequest,
+ schemas.AudioRequest,
+ schemas.Generation3DRequest,
+ schemas.VisionRequest,
+ schemas.MultimodalEmbeddingRequest,
+ ):
+ assert cls in schemas.SUPPORTED_LLM_REQUEST_SCHEMAS
+
+
+def test_mixin_configs_has_nine_entries():
+ m = _BaseLLMMixin()
+ assert len(m._llm_configs) == 9
+
+
+def test_mixin_configs_maps_new_requests_to_responses():
+ m = _BaseLLMMixin()
+ expected = {
+ schemas.ImageGenerationRequest: (schemas.ImageGenerationResponse, "image_generation"),
+ schemas.VideoGenerationRequest: (schemas.VideoGenerationResponse, "video_generation"),
+ schemas.AudioRequest: (schemas.AudioResponse, "audio"),
+ schemas.Generation3DRequest: (schemas.Generation3DResponse, "generation_3d"),
+ schemas.VisionRequest: (schemas.VisionResponse, "vision"),
+ schemas.MultimodalEmbeddingRequest: (schemas.MultimodalEmbeddingResponse, "embedding_multimodal"),
+ }
+ for req_cls, expected_cfg in expected.items():
+ assert m._llm_configs[req_cls] == expected_cfg
+
+
+# ----------------------------------------------------------------------
+# Schema field defaults and required-field validation
+# ----------------------------------------------------------------------
+
+def test_image_generation_request_minimal_valid():
+ r = schemas.ImageGenerationRequest(model="flux", prompt="a cat")
+ assert r.model == "flux"
+ assert r.prompt == "a cat"
+ assert r.num_images == 1 # documented default
+ assert r.image is None
+ assert r.mask is None
+
+
+def test_image_generation_request_missing_prompt_raises():
+ with pytest.raises(ValidationError):
+ schemas.ImageGenerationRequest(model="flux")
+
+
+def test_video_generation_request_defaults():
+ r = schemas.VideoGenerationRequest(model="hunyuan", prompt="x")
+ assert r.duration_s == 5.0
+ assert r.fps == 24
+
+
+def test_audio_request_all_inputs_optional_except_model():
+ r = schemas.AudioRequest(model="whisper")
+ assert r.text is None
+ assert r.audio is None
+ assert r.voice is None
+
+
+def test_generation_3d_request_output_format_default():
+ r = schemas.Generation3DRequest(model="triposr")
+ assert r.output_format == "glb"
+
+
+def test_vision_request_requires_image():
+ with pytest.raises(ValidationError):
+ schemas.VisionRequest(model="clip")
+
+
+def test_multimodal_embedding_request_all_inputs_optional():
+ r = schemas.MultimodalEmbeddingRequest(model="clip")
+ assert r.input is None
+ assert r.image is None
+ assert r.audio is None
+
+
+# ----------------------------------------------------------------------
+# Response shape validation (Literal["..."] guards)
+# ----------------------------------------------------------------------
+
+def test_image_generation_response_object_literal():
+ r = schemas.ImageGenerationResponse(
+ id="imggen-1-abc",
+ object="image_generation",
+ created=1,
+ model="flux",
+ data=[schemas.ImageGenerationData(url="http://x/i.png")],
+ )
+ assert r.object == "image_generation"
+
+
+def test_image_generation_response_wrong_object_raises():
+ with pytest.raises(ValidationError):
+ schemas.ImageGenerationResponse(
+ id="imggen-1-abc",
+ object="wrong",
+ created=1,
+ model="flux",
+ data=[],
+ )
+
+
+def test_multimodal_embedding_response_object_is_list():
+ """Like EmbeddingResponse: outer object is 'list', each data item is 'embedding'."""
+ r = schemas.MultimodalEmbeddingResponse(
+ object="list",
+ data=[
+ schemas.MultimodalEmbeddingData(
+ embedding=[0.1, 0.2],
+ index=0,
+ modality="text",
+ )
+ ],
+ model="clip",
+ )
+ assert r.object == "list"
+ assert r.data[0].object == "embedding" # default on the data item
+
+
+# ----------------------------------------------------------------------
+# Mixin wrap path — covers FRICTION #15 fix + each new endpoint_type
+# ----------------------------------------------------------------------
+
+@pytest.fixture
+def mixin():
+ return _BaseLLMMixin()
+
+
+def test_friction_15_embedding_wrap_outer_object_is_list(mixin):
+ """
+ FRICTION #15 regression. Pre-fix this raised ValidationError because the
+ mixin passed object='embedding' to EmbeddingResponse, which declares
+ object: Literal['list']. After the fix the wrap path succeeds and the
+ outer object is 'list'.
+ """
+ req = schemas.EmbeddingRequest(model="bge", input="hello")
+ wrapped = mixin._wrap_llm_response(
+ result={
+ "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}],
+ "usage": {"prompt_tokens": 1, "total_tokens": 1},
+ },
+ response_model=schemas.EmbeddingResponse,
+ endpoint_type="embedding",
+ openai_req=req,
+ )
+ assert wrapped.object == "list"
+ assert wrapped.data[0].object == "embedding"
+
+
+def test_image_generation_wrap(mixin):
+ req = schemas.ImageGenerationRequest(model="flux", prompt="cat")
+ wrapped = mixin._wrap_llm_response(
+ result={"data": [{"url": "http://x/i.png", "seed": 42}]},
+ response_model=schemas.ImageGenerationResponse,
+ endpoint_type="image_generation",
+ openai_req=req,
+ )
+ assert wrapped.object == "image_generation"
+ assert wrapped.data[0].url == "http://x/i.png"
+ assert wrapped.id.startswith("imggen-")
+
+
+def test_video_generation_wrap(mixin):
+ req = schemas.VideoGenerationRequest(model="hunyuan", prompt="x")
+ wrapped = mixin._wrap_llm_response(
+ result={"data": [{"url": "http://x/v.mp4", "duration_s": 5.0}]},
+ response_model=schemas.VideoGenerationResponse,
+ endpoint_type="video_generation",
+ openai_req=req,
+ )
+ assert wrapped.object == "video_generation"
+ assert wrapped.id.startswith("vidgen-")
+
+
+def test_audio_wrap(mixin):
+ req = schemas.AudioRequest(model="whisper", text="hi")
+ wrapped = mixin._wrap_llm_response(
+ result={"data": [{"text": "hola", "language": "es"}]},
+ response_model=schemas.AudioResponse,
+ endpoint_type="audio",
+ openai_req=req,
+ )
+ assert wrapped.object == "audio"
+ assert wrapped.data[0].text == "hola"
+
+
+def test_generation_3d_wrap(mixin):
+ req = schemas.Generation3DRequest(model="triposr", prompt="chair")
+ wrapped = mixin._wrap_llm_response(
+ result={"data": [{"url": "http://x/c.glb", "output_format": "glb"}]},
+ response_model=schemas.Generation3DResponse,
+ endpoint_type="generation_3d",
+ openai_req=req,
+ )
+ assert wrapped.object == "generation_3d"
+ assert wrapped.id.startswith("gen3d-")
+
+
+def test_multimodal_embedding_wrap(mixin):
+ req = schemas.MultimodalEmbeddingRequest(model="clip", input="hi")
+ wrapped = mixin._wrap_llm_response(
+ result={
+ "data": [
+ {
+ "object": "embedding",
+ "embedding": [0.1, 0.2],
+ "index": 0,
+ "modality": "text",
+ }
+ ]
+ },
+ response_model=schemas.MultimodalEmbeddingResponse,
+ endpoint_type="embedding_multimodal",
+ openai_req=req,
+ )
+ assert wrapped.object == "list"
+ assert wrapped.data[0].embedding == [0.1, 0.2]
+
+
+def test_unknown_endpoint_type_still_raises(mixin):
+ req = schemas.ImageGenerationRequest(model="x", prompt="y")
+ with pytest.raises(ValueError, match="Unknown endpoint type"):
+ mixin._wrap_llm_response(
+ result={"data": []},
+ response_model=schemas.ImageGenerationResponse,
+ endpoint_type="not_a_real_type",
+ openai_req=req,
+ )
From b8cac70741ab2b9eb7c40ae1dca33bf6eb1cfc01 Mon Sep 17 00:00:00 2001
From: nataliasocaity
Date: Mon, 8 Jun 2026 11:06:46 +0100
Subject: [PATCH 02/38] test-services: minimal harness for PR #15 media-file
integration check
Adds two files under examples/ to validate the new six Pydantic schemas
end-to-end as Matthias requested on PR #15:
- examples/test_service_six_schemas.py: APIPod service exposing
/vision (VisionRequest, image required) and /audio (AudioRequest,
audio optional). Each endpoint echoes media properties so we can
see if the file actually got deserialized into ImageFile/AudioFile.
- examples/test_clients_six_schemas.py: runs three checks against
the running service: HTTP request with base64, HTTP request with URL,
and a fastSDK call via APISeex.
Result on this branch (server starts cleanly):
- HTTP base64: 422 "Input should be an instance of ImageFile"
- HTTP URL: 422 same error
- fastSDK: does not dispatch the request without a registered
service (no ad-hoc URL path in FastClient / create_sdk)
Confirms Matthias's hypothesis that media-file integration with the
new schemas does not work out of the box. Root cause is that the
fastapi file_handling_mixin converts URL/base64 to ImageFile for
*loose endpoint parameters*, not for ImageFile fields declared inside
a Pydantic body model. Fix is out of scope of this PR.
---
examples/test_clients_six_schemas.py | 168 +++++++++++++++++++++++++++
examples/test_service_six_schemas.py | 76 ++++++++++++
2 files changed, 244 insertions(+)
create mode 100644 examples/test_clients_six_schemas.py
create mode 100644 examples/test_service_six_schemas.py
diff --git a/examples/test_clients_six_schemas.py b/examples/test_clients_six_schemas.py
new file mode 100644
index 0000000..51fb365
--- /dev/null
+++ b/examples/test_clients_six_schemas.py
@@ -0,0 +1,168 @@
+"""
+Exercises the test service in three ways, as Matthias asked on PR #15:
+ 1. fastSDK client
+ 2. HTTP request with base64
+ 3. HTTP request with URL
+
+Run the service first in another shell:
+ python examples/test_service_six_schemas.py
+
+Then run this:
+ python examples/test_clients_six_schemas.py
+"""
+
+import base64
+import io
+import json
+import sys
+import urllib.request
+from pathlib import Path
+
+import requests
+
+BASE_URL = "http://localhost:8000"
+# A small public PNG used for the "URL" test. Picked because it doesn't need
+# auth and returns a real PNG header (so media_toolkit can sniff it).
+PUBLIC_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/120px-PNG_transparency_demonstration_1.png"
+
+
+def _make_local_png_bytes() -> bytes:
+ """A minimal valid PNG, no dependencies."""
+ return bytes.fromhex(
+ "89504e470d0a1a0a0000000d49484452"
+ "000000010000000108060000001f15c4"
+ "890000000a49444154789c6300010000"
+ "0500010d0a2db40000000049454e44ae"
+ "426082"
+ )
+
+
+def _print_result(label: str, status: int, body) -> None:
+ print(f"\n=== {label} ===")
+ print(f"HTTP {status}")
+ print(json.dumps(body, indent=2) if isinstance(body, (dict, list)) else body)
+
+
+# -------------------------------------------------------------------
+# 1. HTTP with base64
+# -------------------------------------------------------------------
+def test_http_base64() -> bool:
+ png = _make_local_png_bytes()
+ b64 = base64.b64encode(png).decode("ascii")
+ payload = {
+ "model": "test-model",
+ "image": b64,
+ }
+ r = requests.post(f"{BASE_URL}/vision", json=payload, timeout=15)
+ try:
+ body = r.json()
+ except Exception:
+ body = r.text
+ _print_result("HTTP base64", r.status_code, body)
+ return r.status_code == 200
+
+
+# -------------------------------------------------------------------
+# 2. HTTP with URL
+# -------------------------------------------------------------------
+def test_http_url() -> bool:
+ payload = {
+ "model": "test-model",
+ "image": PUBLIC_IMAGE_URL,
+ }
+ r = requests.post(f"{BASE_URL}/vision", json=payload, timeout=30)
+ try:
+ body = r.json()
+ except Exception:
+ body = r.text
+ _print_result("HTTP URL", r.status_code, body)
+ return r.status_code == 200
+
+
+# -------------------------------------------------------------------
+# 3. fastSDK client
+# -------------------------------------------------------------------
+def test_fastsdk() -> bool:
+ """
+ Hits /vision through fastSDK.
+
+ Note: fastSDK's public entry points (`FastClient`, `create_sdk`,
+ `APISeex`) all expect either a service registered in the Registry
+ or a generated client class. There is no ad-hoc "given this URL,
+ just POST this payload" path. For the purposes of PR #15 this means
+ a single localhost service like the one in this example is not the
+ intended fastSDK use case; the right shape would be to register
+ the service first and then call it through the SDK.
+
+ For now we just dispatch a low-level APISeex with a minimal
+ ServiceDefinition and report whatever happens.
+ """
+ try:
+ from fastsdk import APISeex, ImageFile
+ from apipod_registry.definitions.service_definitions import (
+ ServiceDefinition, EndpointDefinition, EndpointParameter,
+ ServiceAddress,
+ )
+ except Exception as exc:
+ _print_result("fastSDK", -1, f"import failed: {exc.__class__.__name__}: {exc}")
+ return False
+
+ try:
+ service = ServiceDefinition(
+ id="test-svc",
+ display_name="test svc",
+ description="local test service",
+ service_address=ServiceAddress(url=BASE_URL),
+ endpoints=[],
+ )
+ endpoint = EndpointDefinition(
+ id="vision",
+ path="/vision",
+ display_name="vision",
+ description="vision echo",
+ parameters=[
+ EndpointParameter(
+ name="model", type="string", required=True,
+ location="body", default="test-model",
+ ),
+ EndpointParameter(
+ name="image", type="image", required=True,
+ location="body",
+ ),
+ ],
+ )
+ png = _make_local_png_bytes()
+ img = ImageFile().from_bytes(png)
+ seex = APISeex(
+ service_def=service,
+ endpoint_def=endpoint,
+ data={"model": "test-model", "image": img},
+ )
+ result = seex.wait_for_result(timeout_s=15)
+ if seex.error:
+ _print_result("fastSDK", -1, f"server returned error: {seex.error}")
+ return False
+ if result is None:
+ _print_result("fastSDK", -1, "no result returned (request likely not dispatched; needs Registry setup)")
+ return False
+ _print_result("fastSDK", 200, str(result))
+ return True
+ except Exception as exc:
+ _print_result("fastSDK", -1, f"call failed: {exc.__class__.__name__}: {exc}")
+ return False
+
+
+def main() -> int:
+ results = {
+ "HTTP base64": test_http_base64(),
+ "HTTP URL": test_http_url(),
+ "fastSDK": test_fastsdk(),
+ }
+ print("\n=== Summary ===")
+ for name, ok in results.items():
+ print(f" {name}: {'PASS' if ok else 'FAIL'}")
+ return 0 if all(results.values()) else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/examples/test_service_six_schemas.py b/examples/test_service_six_schemas.py
new file mode 100644
index 0000000..2564bcf
--- /dev/null
+++ b/examples/test_service_six_schemas.py
@@ -0,0 +1,76 @@
+"""
+Minimal APIPod service used to verify media-file integration end to end for
+PR #15 (the 6 new Pydantic schemas).
+
+It exposes two endpoints:
+- POST /vision uses VisionRequest, which has a REQUIRED image field
+- POST /audio uses AudioRequest, which has an OPTIONAL audio field
+
+Both endpoints just inspect the incoming media file and echo back a few
+properties (size, content type, etc.) so we can confirm the file_handling
+mixin actually deserialized URL/base64 input into a real ImageFile/AudioFile.
+
+Run with:
+ python examples/test_service_six_schemas.py
+The server starts on http://0.0.0.0:8000 with OpenAPI docs at /docs.
+"""
+
+from apipod import APIPod
+from apipod.common.schemas import (
+ VisionRequest,
+ VisionResponse,
+ VisionData,
+ AudioRequest,
+ AudioResponse,
+ AudioData,
+)
+
+
+app = APIPod()
+
+
+def _media_summary(media) -> dict:
+ """Pull a couple of safe fields off whatever media_toolkit returned."""
+ if media is None:
+ return {"present": False}
+ return {
+ "present": True,
+ "type": type(media).__name__,
+ "size_bytes": getattr(media, "file_size", None) or getattr(media, "size", None),
+ "content_type": getattr(media, "content_type", None),
+ }
+
+
+@app.endpoint("/vision")
+def vision(req: VisionRequest):
+ """
+ Echoes back a fake VisionData. The point is not the label, it's that
+ req.image arrives as an ImageFile (deserialized from upload / base64 / URL).
+ """
+ summary = _media_summary(req.image)
+ return {
+ "data": [
+ VisionData(label=f"echo:{summary}", score=1.0, box=None)
+ ]
+ }
+
+
+@app.endpoint("/audio")
+def audio(req: AudioRequest):
+ """
+ Echoes the audio properties (or text if no audio was sent).
+ """
+ if req.audio is not None:
+ summary = _media_summary(req.audio)
+ text = f"echo:{summary}"
+ else:
+ text = f"no-audio:text={req.text!r}"
+ return {
+ "data": [
+ AudioData(text=text, language=req.language, duration_s=None)
+ ]
+ }
+
+
+if __name__ == "__main__":
+ app.start(port=8000, host="0.0.0.0")
From e932d0546e194af2221702b98564934024033cc9 Mon Sep 17 00:00:00 2001
From: nataliasocaity
Date: Mon, 8 Jun 2026 13:17:10 +0100
Subject: [PATCH 03/38] schemas: rename OpenAIBaseModel to APIPodSchemaBase
Per Carlos's review on PR #15: the previous name suggested the schemas
were tied to OpenAI's own models, when the intent is the opposite. The
shape mirrors OpenAI's API for client compatibility, but any provider
(Flux, Stable Diffusion, ElevenLabs, Whisper, etc.) can sit behind it.
Renames the base class and adds a module-level + class-level docstring
making the provider-agnostic intent explicit. No behaviour change.
The related concerns about apipod/engine/llm/_BaseLLMMixin (used for
non-LLM endpoints like image/audio/video generation) are not addressed
here on purpose, because that rename touches the runpod and fastapi
backends and their tests. Tracked for a separate follow-up PR.
---
apipod/common/schemas.py | 85 +++++++++++++++++++++++++---------------
1 file changed, 53 insertions(+), 32 deletions(-)
diff --git a/apipod/common/schemas.py b/apipod/common/schemas.py
index 89e664a..f111221 100644
--- a/apipod/common/schemas.py
+++ b/apipod/common/schemas.py
@@ -1,3 +1,15 @@
+"""
+Standard request / response schemas for APIPod services.
+
+The shape of every schema in this file mirrors the OpenAI API so that
+clients written against the OpenAI SDK (or any OpenAI-compatible tool)
+can talk to an APIPod service without translation. That choice is about
+the wire format; it does NOT imply the schemas are tied to OpenAI's own
+models. Any provider (Flux, Stable Diffusion, ElevenLabs, Whisper, Suno,
+DeepSeek, etc.) plugs into the same schemas and the routing layer
+dispatches to whatever runs behind it.
+"""
+
from typing import List, Optional, Union, Literal
from pydantic import BaseModel
@@ -7,7 +19,16 @@
# Base schema
# =====================================================
-class OpenAIBaseModel(BaseModel):
+class APIPodSchemaBase(BaseModel):
+ """
+ Base for every APIPod request / response model.
+
+ Shape follows the OpenAI API spec so OpenAI-compatible clients work
+ out of the box, but the schemas are provider-agnostic: anything that
+ matches the shape (Flux, SD, ElevenLabs, Whisper, in-house models...)
+ is a valid backend.
+ """
+
model_config = {
"extra": "forbid",
"validate_assignment": True,
@@ -23,12 +44,12 @@ class OpenAIBaseModel(BaseModel):
# Chat Completions - Input schemas
# =====================================================
-class ChatMessage(OpenAIBaseModel):
+class ChatMessage(APIPodSchemaBase):
role: Literal["system", "user", "assistant"]
content: str
-class ChatCompletionRequest(OpenAIBaseModel):
+class ChatCompletionRequest(APIPodSchemaBase):
model: str
messages: List[ChatMessage]
@@ -47,7 +68,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
# Text Completion - Input schemas
# =====================================================
-class CompletionRequest(OpenAIBaseModel):
+class CompletionRequest(APIPodSchemaBase):
model: str
prompt: Union[str, List[str]]
@@ -63,7 +84,7 @@ class CompletionRequest(OpenAIBaseModel):
# Embedding - Input schemas
# =====================================================
-class EmbeddingRequest(OpenAIBaseModel):
+class EmbeddingRequest(APIPodSchemaBase):
model: str
input: Union[str, List[str]]
user: Optional[str] = None
@@ -73,7 +94,7 @@ class EmbeddingRequest(OpenAIBaseModel):
# Image Generation - Input schemas
# =====================================================
-class ImageGenerationRequest(OpenAIBaseModel):
+class ImageGenerationRequest(APIPodSchemaBase):
model: str
prompt: str
@@ -90,7 +111,7 @@ class ImageGenerationRequest(OpenAIBaseModel):
# Video Generation - Input schemas
# =====================================================
-class VideoGenerationRequest(OpenAIBaseModel):
+class VideoGenerationRequest(APIPodSchemaBase):
model: str
prompt: str
@@ -105,7 +126,7 @@ class VideoGenerationRequest(OpenAIBaseModel):
# Audio - Input schemas (TTS, STT, music)
# =====================================================
-class AudioRequest(OpenAIBaseModel):
+class AudioRequest(APIPodSchemaBase):
model: str
text: Optional[str] = None
@@ -120,7 +141,7 @@ class AudioRequest(OpenAIBaseModel):
# 3D Generation - Input schemas
# =====================================================
-class Generation3DRequest(OpenAIBaseModel):
+class Generation3DRequest(APIPodSchemaBase):
model: str
prompt: Optional[str] = None
@@ -133,7 +154,7 @@ class Generation3DRequest(OpenAIBaseModel):
# Vision - Input schemas (classify, detect, OCR)
# =====================================================
-class VisionRequest(OpenAIBaseModel):
+class VisionRequest(APIPodSchemaBase):
model: str
image: ImageFile
@@ -146,7 +167,7 @@ class VisionRequest(OpenAIBaseModel):
# Multimodal Embedding - Input schemas
# =====================================================
-class MultimodalEmbeddingRequest(OpenAIBaseModel):
+class MultimodalEmbeddingRequest(APIPodSchemaBase):
model: str
input: Optional[Union[str, List[str]]] = None
@@ -174,7 +195,7 @@ class MultimodalEmbeddingRequest(OpenAIBaseModel):
# Shared output schemas
# =====================================================
-class Usage(OpenAIBaseModel):
+class Usage(APIPodSchemaBase):
prompt_tokens: int
completion_tokens: Optional[int] = None
total_tokens: int
@@ -184,18 +205,18 @@ class Usage(OpenAIBaseModel):
# Chat Completions - Output schemas
# =====================================================
-class ChatCompletionMessage(OpenAIBaseModel):
+class ChatCompletionMessage(APIPodSchemaBase):
role: Literal["assistant"]
content: str
-class ChatCompletionChoice(OpenAIBaseModel):
+class ChatCompletionChoice(APIPodSchemaBase):
index: int
message: ChatCompletionMessage
finish_reason: Literal["stop", "length", "content_filter"]
-class ChatCompletionResponse(OpenAIBaseModel):
+class ChatCompletionResponse(APIPodSchemaBase):
id: str
object: Literal["chat.completion"]
created: int
@@ -208,14 +229,14 @@ class ChatCompletionResponse(OpenAIBaseModel):
# Text Completion - Output schemas
# =====================================================
-class CompletionChoice(OpenAIBaseModel):
+class CompletionChoice(APIPodSchemaBase):
text: str
index: int
logprobs: None = None
finish_reason: Literal["stop", "length", "content_filter"]
-class CompletionResponse(OpenAIBaseModel):
+class CompletionResponse(APIPodSchemaBase):
id: str
object: Literal["text_completion"]
created: int
@@ -228,13 +249,13 @@ class CompletionResponse(OpenAIBaseModel):
# Embedding - Output schemas
# =====================================================
-class EmbeddingData(OpenAIBaseModel):
+class EmbeddingData(APIPodSchemaBase):
object: Literal["embedding"]
embedding: List[float]
index: int
-class EmbeddingResponse(OpenAIBaseModel):
+class EmbeddingResponse(APIPodSchemaBase):
object: Literal["list"]
data: List[EmbeddingData]
model: str
@@ -245,14 +266,14 @@ class EmbeddingResponse(OpenAIBaseModel):
# Image Generation - Output schemas
# =====================================================
-class ImageGenerationData(OpenAIBaseModel):
+class ImageGenerationData(APIPodSchemaBase):
url: Optional[str] = None
b64_json: Optional[str] = None
revised_prompt: Optional[str] = None
seed: Optional[int] = None
-class ImageGenerationResponse(OpenAIBaseModel):
+class ImageGenerationResponse(APIPodSchemaBase):
id: str
object: Literal["image_generation"]
created: int
@@ -265,13 +286,13 @@ class ImageGenerationResponse(OpenAIBaseModel):
# Video Generation - Output schemas
# =====================================================
-class VideoGenerationData(OpenAIBaseModel):
+class VideoGenerationData(APIPodSchemaBase):
url: Optional[str] = None
duration_s: Optional[float] = None
seed: Optional[int] = None
-class VideoGenerationResponse(OpenAIBaseModel):
+class VideoGenerationResponse(APIPodSchemaBase):
id: str
object: Literal["video_generation"]
created: int
@@ -284,14 +305,14 @@ class VideoGenerationResponse(OpenAIBaseModel):
# Audio - Output schemas
# =====================================================
-class AudioData(OpenAIBaseModel):
+class AudioData(APIPodSchemaBase):
audio: Optional[str] = None
text: Optional[str] = None
language: Optional[str] = None
duration_s: Optional[float] = None
-class AudioResponse(OpenAIBaseModel):
+class AudioResponse(APIPodSchemaBase):
id: str
object: Literal["audio"]
created: int
@@ -304,13 +325,13 @@ class AudioResponse(OpenAIBaseModel):
# 3D Generation - Output schemas
# =====================================================
-class Generation3DData(OpenAIBaseModel):
+class Generation3DData(APIPodSchemaBase):
url: Optional[str] = None
output_format: Optional[str] = None
seed: Optional[int] = None
-class Generation3DResponse(OpenAIBaseModel):
+class Generation3DResponse(APIPodSchemaBase):
id: str
object: Literal["generation_3d"]
created: int
@@ -323,18 +344,18 @@ class Generation3DResponse(OpenAIBaseModel):
# Vision - Output schemas (classify, detect, OCR)
# =====================================================
-class VisionLabel(OpenAIBaseModel):
+class VisionLabel(APIPodSchemaBase):
label: str
score: float
box: Optional[List[float]] = None
-class VisionData(OpenAIBaseModel):
+class VisionData(APIPodSchemaBase):
labels: List[VisionLabel] = []
text: Optional[str] = None
-class VisionResponse(OpenAIBaseModel):
+class VisionResponse(APIPodSchemaBase):
id: str
object: Literal["vision"]
created: int
@@ -347,14 +368,14 @@ class VisionResponse(OpenAIBaseModel):
# Multimodal Embedding - Output schemas
# =====================================================
-class MultimodalEmbeddingData(OpenAIBaseModel):
+class MultimodalEmbeddingData(APIPodSchemaBase):
object: Literal["embedding"] = "embedding"
embedding: List[float]
index: int
modality: Optional[Literal["text", "image", "audio"]] = None
-class MultimodalEmbeddingResponse(OpenAIBaseModel):
+class MultimodalEmbeddingResponse(APIPodSchemaBase):
object: Literal["list"]
data: List[MultimodalEmbeddingData]
model: str
From 8549eab78c037d19679ab8e35307bd95c303a794 Mon Sep 17 00:00:00 2001
From: nataliasocaity
Date: Tue, 9 Jun 2026 08:15:47 +0100
Subject: [PATCH 04/38] schemas: pre-validator to convert URL / base64 strings
into media files inside Pydantic bodies
Matthias asked on PR #15 to make media-file integration actually work for
the new schemas with JSON callers sending base64 or URLs. file_handling_mixin
converts those for loose endpoint parameters but not for ImageFile / AudioFile
/ VideoFile / MediaFile fields declared inside a Pydantic body model, where
Pydantic validates the string before the mixin gets a chance.
This adds a model_validator(mode='before') on APIPodSchemaBase that walks
model_fields, picks up media-typed fields, and converts string values:
URLs and data URIs go through media_from_any, bare base64 strings go through
the file class's from_base64 (media_from_any does not recognise bare base64).
Non-string values pass through untouched so existing callers that already
hand over an ImageFile instance keep working.
E2E (examples/test_service_six_schemas.py + test_clients_six_schemas.py):
HTTP base64 PASS, HTTP URL PASS, both return 200 with ImageFile in the echo.
fastSDK still requires Registry setup as called out previously, no change.
The 21 unit tests in test/llm/test_six_schemas.py still pass.
examples/test_service_six_schemas.py: param renamed to 'payload' to match the
router convention, response dicts use the actual VisionData / AudioData
shapes (labels: List[VisionLabel] for vision, audio fields for audio).
---
apipod/common/schemas.py | 54 ++++++++++++++++++++++++----
examples/test_service_six_schemas.py | 34 ++++++++----------
2 files changed, 62 insertions(+), 26 deletions(-)
diff --git a/apipod/common/schemas.py b/apipod/common/schemas.py
index f111221..413aa35 100644
--- a/apipod/common/schemas.py
+++ b/apipod/common/schemas.py
@@ -10,10 +10,25 @@
dispatches to whatever runs behind it.
"""
-from typing import List, Optional, Union, Literal
-from pydantic import BaseModel
+from types import UnionType
+from typing import Any, List, Literal, Optional, Union, get_args, get_origin
+from pydantic import BaseModel, model_validator
+
+from media_toolkit import AudioFile, ImageFile, MediaFile, VideoFile, media_from_any
+
+_MEDIA_FIELD_TYPES = (ImageFile, AudioFile, VideoFile, MediaFile)
+
+
+def _media_field_type(annotation: Any) -> Optional[type]:
+ """Media class if annotation is media-typed (bare, Optional, or Union), else None."""
+ if annotation in _MEDIA_FIELD_TYPES:
+ return annotation
+ if get_origin(annotation) in (Union, UnionType):
+ for arg in get_args(annotation):
+ if arg in _MEDIA_FIELD_TYPES:
+ return arg
+ return None
-from media_toolkit import ImageFile, AudioFile
# =====================================================
# Base schema
@@ -33,12 +48,39 @@ class APIPodSchemaBase(BaseModel):
"extra": "forbid",
"validate_assignment": True,
"populate_by_name": True,
- # Allow media_toolkit file types (ImageFile, AudioFile, VideoFile, MediaFile)
- # as field annotations. APIPod's file_handling_mixin converts URL/base64
- # inputs into these types before request validation.
+ # ImageFile / AudioFile / VideoFile / MediaFile are non-BaseModel types
+ # used as field annotations; the pre-validator below converts URL /
+ # base64 strings into instances before per-field validation.
"arbitrary_types_allowed": True,
}
+ @model_validator(mode="before")
+ @classmethod
+ def _convert_media_strings(cls, data: Any) -> Any:
+ """URL / data URI through media_from_any, bare base64 through from_base64."""
+ if not isinstance(data, dict):
+ return data
+ for field_name, field_info in cls.model_fields.items():
+ if field_name not in data:
+ continue
+ value = data[field_name]
+ if not isinstance(value, str):
+ continue
+ media_type = _media_field_type(field_info.annotation)
+ if media_type is None:
+ continue
+ if value.startswith(("http://", "https://", "data:")):
+ data[field_name] = media_from_any(
+ data=value,
+ type_hint=media_type,
+ use_temp_file=True,
+ temp_dir=None,
+ allow_reads_from_disk=False,
+ )
+ else:
+ data[field_name] = media_type().from_base64(value)
+ return data
+
# =====================================================
# Chat Completions - Input schemas
diff --git a/examples/test_service_six_schemas.py b/examples/test_service_six_schemas.py
index 2564bcf..9d0d36d 100644
--- a/examples/test_service_six_schemas.py
+++ b/examples/test_service_six_schemas.py
@@ -18,11 +18,7 @@
from apipod import APIPod
from apipod.common.schemas import (
VisionRequest,
- VisionResponse,
- VisionData,
AudioRequest,
- AudioResponse,
- AudioData,
)
@@ -42,32 +38,30 @@ def _media_summary(media) -> dict:
@app.endpoint("/vision")
-def vision(req: VisionRequest):
- """
- Echoes back a fake VisionData. The point is not the label, it's that
- req.image arrives as an ImageFile (deserialized from upload / base64 / URL).
- """
- summary = _media_summary(req.image)
+def vision(payload: VisionRequest):
+ """Echoes a label built from payload.image. The point is that the image
+ arrived as an ImageFile, deserialized from upload / base64 / URL."""
+ summary = _media_summary(payload.image)
return {
"data": [
- VisionData(label=f"echo:{summary}", score=1.0, box=None)
+ {
+ "labels": [{"label": f"echo:{summary}", "score": 1.0}],
+ "text": None,
+ }
]
}
@app.endpoint("/audio")
-def audio(req: AudioRequest):
- """
- Echoes the audio properties (or text if no audio was sent).
- """
- if req.audio is not None:
- summary = _media_summary(req.audio)
- text = f"echo:{summary}"
+def audio(payload: AudioRequest):
+ """Echoes the audio properties (or text if no audio was sent)."""
+ if payload.audio is not None:
+ text = f"echo:{_media_summary(payload.audio)}"
else:
- text = f"no-audio:text={req.text!r}"
+ text = f"no-audio:text={payload.text!r}"
return {
"data": [
- AudioData(text=text, language=req.language, duration_s=None)
+ {"text": text, "language": payload.language, "duration_s": None}
]
}
From 6932f0941546c6d523a8195f7801be738496ae8b Mon Sep 17 00:00:00 2001
From: nataliasocaity
Date: Tue, 9 Jun 2026 08:54:56 +0100
Subject: [PATCH 05/38] test: fill coverage gaps for the six new modality
schemas
Previous test file had 21 tests but coverage was uneven across modalities.
Vision in particular had no wrap test, so the vision elif branch in
_wrap_llm_response was never executed.
Adds 10 tests to bring video / audio / 3d / vision to parity with image:
- video missing-prompt validation
- response object Literal["..."] check + wrong_object_raises for video,
audio, 3d, vision
- vision_wrap covering the previously unexercised elif branch
Total: 21 -> 31, all passing.
---
test/llm/test_six_schemas.py | 124 +++++++++++++++++++++++++++++++++++
1 file changed, 124 insertions(+)
diff --git a/test/llm/test_six_schemas.py b/test/llm/test_six_schemas.py
index d4f7a06..cdd6a6e 100644
--- a/test/llm/test_six_schemas.py
+++ b/test/llm/test_six_schemas.py
@@ -77,6 +77,11 @@ def test_video_generation_request_defaults():
assert r.fps == 24
+def test_video_generation_request_missing_prompt_raises():
+ with pytest.raises(ValidationError):
+ schemas.VideoGenerationRequest(model="hunyuan")
+
+
def test_audio_request_all_inputs_optional_except_model():
r = schemas.AudioRequest(model="whisper")
assert r.text is None
@@ -127,6 +132,94 @@ def test_image_generation_response_wrong_object_raises():
)
+def test_video_generation_response_object_literal():
+ r = schemas.VideoGenerationResponse(
+ id="vidgen-1-abc",
+ object="video_generation",
+ created=1,
+ model="hunyuan",
+ data=[schemas.VideoGenerationData(url="http://x/v.mp4")],
+ )
+ assert r.object == "video_generation"
+
+
+def test_video_generation_response_wrong_object_raises():
+ with pytest.raises(ValidationError):
+ schemas.VideoGenerationResponse(
+ id="vidgen-1-abc",
+ object="wrong",
+ created=1,
+ model="hunyuan",
+ data=[],
+ )
+
+
+def test_audio_response_object_literal():
+ r = schemas.AudioResponse(
+ id="aud-1-abc",
+ object="audio",
+ created=1,
+ model="whisper",
+ data=[schemas.AudioData(text="hola")],
+ )
+ assert r.object == "audio"
+
+
+def test_audio_response_wrong_object_raises():
+ with pytest.raises(ValidationError):
+ schemas.AudioResponse(
+ id="aud-1-abc",
+ object="wrong",
+ created=1,
+ model="whisper",
+ data=[],
+ )
+
+
+def test_generation_3d_response_object_literal():
+ r = schemas.Generation3DResponse(
+ id="gen3d-1-abc",
+ object="generation_3d",
+ created=1,
+ model="triposr",
+ data=[schemas.Generation3DData(url="http://x/c.glb")],
+ )
+ assert r.object == "generation_3d"
+
+
+def test_generation_3d_response_wrong_object_raises():
+ with pytest.raises(ValidationError):
+ schemas.Generation3DResponse(
+ id="gen3d-1-abc",
+ object="wrong",
+ created=1,
+ model="triposr",
+ data=[],
+ )
+
+
+def test_vision_response_object_literal():
+ r = schemas.VisionResponse(
+ id="vis-1-abc",
+ object="vision",
+ created=1,
+ model="clip",
+ data=[schemas.VisionData(labels=[])],
+ )
+ assert r.object == "vision"
+
+
+def test_vision_response_wrong_object_raises():
+ with pytest.raises(ValidationError):
+ schemas.VisionResponse(
+ id="vis-1-abc",
+ object="wrong",
+ created=1,
+ model="clip",
+ data=[],
+ )
+
+
def test_multimodal_embedding_response_object_is_list():
"""Like EmbeddingResponse: outer object is 'list', each data item is 'embedding'."""
r = schemas.MultimodalEmbeddingResponse(
@@ -223,6 +316,37 @@ def test_generation_3d_wrap(mixin):
assert wrapped.id.startswith("gen3d-")
+# 1x1 transparent PNG used to construct ImageFile without hitting the network.
+_TINY_PNG_B64 = (
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
+)
+
+
+def test_vision_wrap(mixin):
+ """
+ Vision branch in _wrap_llm_response had no test before; the elif path was
+ never exercised. Covers the branch end-to-end.
+ """
+ req = schemas.VisionRequest(model="clip", image=_TINY_PNG_B64)
+ wrapped = mixin._wrap_llm_response(
+ result={
+ "data": [
+ {
+ "labels": [{"label": "cat", "score": 0.95}],
+ "text": None,
+ }
+ ]
+ },
+ response_model=schemas.VisionResponse,
+ endpoint_type="vision",
+ openai_req=req,
+ )
+ assert wrapped.object == "vision"
+ assert wrapped.id.startswith("vis-")
+ assert wrapped.data[0].labels[0].label == "cat"
+ assert wrapped.data[0].labels[0].score == 0.95
+
+
def test_multimodal_embedding_wrap(mixin):
req = schemas.MultimodalEmbeddingRequest(model="clip", input="hi")
wrapped = mixin._wrap_llm_response(
From 93fcb877b5e9a746dd32fdc7bf1a2b400740236d Mon Sep 17 00:00:00 2001
From: nataliasocaity
Date: Tue, 9 Jun 2026 09:14:48 +0100
Subject: [PATCH 06/38] schemas: drop the URL/base64 routing in the media
pre-validator
Matthias pointed out that media_from_any already detects base64 without
a data: prefix. My earlier sniff was confused by a 1x1 test PNG (~67
bytes / 92 b64 chars), which sits below media_toolkit's sniffer length
threshold (it bails on short payloads to avoid false positives on
identifier-looking strings). With a realistic payload the heuristic
kicks in and a single media_from_any call covers URL, data URI, and
bare base64.
- pre-validator on APIPodSchemaBase now always routes through media_from_any.
The branch on ("http://", "https://", "data:") and the from_base64
fallback are gone.
- test_clients_six_schemas.py: build a 32x32 PNG (~100 bytes / ~140 b64
chars) instead of the 1x1, so the E2E base64 path exercises the same
shape a realistic JSON caller sends.
- test_six_schemas.py: replace the inline 1x1 _TINY_PNG_B64 constant with a
helper that builds the 32x32 PNG; documents the threshold reason inline.
E2E still passes (HTTP base64 200, HTTP URL 200, fastSDK still gated by
Registry setup as flagged before). All 31 unit tests pass.
---
apipod/common/schemas.py | 17 +++++++--------
examples/test_clients_six_schemas.py | 31 +++++++++++++++++++++-------
test/llm/test_six_schemas.py | 28 +++++++++++++++++++++----
3 files changed, 55 insertions(+), 21 deletions(-)
diff --git a/apipod/common/schemas.py b/apipod/common/schemas.py
index 413aa35..d8216a2 100644
--- a/apipod/common/schemas.py
+++ b/apipod/common/schemas.py
@@ -69,16 +69,13 @@ def _convert_media_strings(cls, data: Any) -> Any:
media_type = _media_field_type(field_info.annotation)
if media_type is None:
continue
- if value.startswith(("http://", "https://", "data:")):
- data[field_name] = media_from_any(
- data=value,
- type_hint=media_type,
- use_temp_file=True,
- temp_dir=None,
- allow_reads_from_disk=False,
- )
- else:
- data[field_name] = media_type().from_base64(value)
+ data[field_name] = media_from_any(
+ data=value,
+ type_hint=media_type,
+ use_temp_file=True,
+ temp_dir=None,
+ allow_reads_from_disk=False,
+ )
return data
diff --git a/examples/test_clients_six_schemas.py b/examples/test_clients_six_schemas.py
index 51fb365..43dc4ba 100644
--- a/examples/test_clients_six_schemas.py
+++ b/examples/test_clients_six_schemas.py
@@ -27,13 +27,30 @@
def _make_local_png_bytes() -> bytes:
- """A minimal valid PNG, no dependencies."""
- return bytes.fromhex(
- "89504e470d0a1a0a0000000d49484452"
- "000000010000000108060000001f15c4"
- "890000000a49444154789c6300010000"
- "0500010d0a2db40000000049454e44ae"
- "426082"
+ """Build a small but real-sized PNG (32x32 solid red) in-process.
+
+ media_toolkit's base64 sniffer needs a payload long enough to be confident
+ it is base64 and not a short identifier-looking string. The original 1x1
+ PNG (67 bytes / 92 b64 chars) sat below the heuristic; 32x32 (~100 bytes /
+ ~140 b64 chars) is comfortably above it and matches realistic JSON callers.
+ """
+ import struct
+ import zlib
+
+ width = height = 32
+ color = (255, 0, 0)
+
+ ihdr_data = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
+ raw = b""
+ for _ in range(height):
+ raw += b"\x00" + bytes(color * width)
+ idat = zlib.compress(raw)
+
+ return (
+ b"\x89PNG\r\n\x1a\n"
+ + struct.pack(">I", len(ihdr_data)) + b"IHDR" + ihdr_data + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr_data))
+ + struct.pack(">I", len(idat)) + b"IDAT" + idat + struct.pack(">I", zlib.crc32(b"IDAT" + idat))
+ + struct.pack(">I", 0) + b"IEND" + struct.pack(">I", zlib.crc32(b"IEND"))
)
diff --git a/test/llm/test_six_schemas.py b/test/llm/test_six_schemas.py
index cdd6a6e..c1dd566 100644
--- a/test/llm/test_six_schemas.py
+++ b/test/llm/test_six_schemas.py
@@ -316,10 +316,30 @@ def test_generation_3d_wrap(mixin):
assert wrapped.id.startswith("gen3d-")
-# 1x1 transparent PNG used to construct ImageFile without hitting the network.
-_TINY_PNG_B64 = (
- "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
-)
+# 32x32 solid-red PNG (~100 bytes / ~140 b64 chars). The previous 1x1 PNG
+# was too short for media_toolkit's base64 sniffer (it bails on payloads
+# below the heuristic length to avoid false positives on short tokens).
+def _make_png_b64() -> str:
+ import base64
+ import struct
+ import zlib
+
+ w = h = 32
+ ihdr = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)
+ raw = b""
+ for _ in range(h):
+ raw += b"\x00" + bytes((255, 0, 0) * w)
+ idat = zlib.compress(raw)
+ png = (
+ b"\x89PNG\r\n\x1a\n"
+ + struct.pack(">I", len(ihdr)) + b"IHDR" + ihdr + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr))
+ + struct.pack(">I", len(idat)) + b"IDAT" + idat + struct.pack(">I", zlib.crc32(b"IDAT" + idat))
+ + struct.pack(">I", 0) + b"IEND" + struct.pack(">I", zlib.crc32(b"IEND"))
+ )
+ return base64.b64encode(png).decode("ascii")
+
+
+_TINY_PNG_B64 = _make_png_b64()
def test_vision_wrap(mixin):
From c0c8892405dfc848ffe759dce1d6e68361734c89 Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Fri, 12 Jun 2026 09:21:39 +0200
Subject: [PATCH 07/38] changes
---
apipod/engine/backend/fastapi/file_handling_mixin.py | 4 ++--
apipod/engine/jobs/job_progress.py | 11 +++++++++--
apipod/engine/queue/job_queue.py | 2 +-
3 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/apipod/engine/backend/fastapi/file_handling_mixin.py b/apipod/engine/backend/fastapi/file_handling_mixin.py
index 0c05d62..fb74aba 100644
--- a/apipod/engine/backend/fastapi/file_handling_mixin.py
+++ b/apipod/engine/backend/fastapi/file_handling_mixin.py
@@ -254,7 +254,7 @@ def _inject_dummy_job_progress(self, func: Callable) -> Callable:
sig = inspect.signature(func)
job_progress_params = [
p.name for p in sig.parameters.values()
- if p.name == "job_progress" or "FastJobProgress" in str(p.annotation)
+ if p.name == "job_progress" or "JobProgress" in str(p.annotation)
]
if not job_progress_params:
@@ -290,7 +290,7 @@ def _remove_job_progress_from_signature(self, func: Callable) -> Callable:
sig = inspect.signature(func)
new_sig = sig.replace(parameters=[
p for p in sig.parameters.values()
- if p.name != "job_progress" and "FastJobProgress" not in str(p.annotation)
+ if p.name != "job_progress" and "JobProgress" not in str(p.annotation)
])
if len(new_sig.parameters) != len(sig.parameters):
return replace_func_signature(func, new_sig)
diff --git a/apipod/engine/jobs/job_progress.py b/apipod/engine/jobs/job_progress.py
index 099e3c9..b9b6ed5 100644
--- a/apipod/engine/jobs/job_progress.py
+++ b/apipod/engine/jobs/job_progress.py
@@ -1,3 +1,7 @@
+import logging
+logger = logging.getLogger(__name__)
+
+
class JobProgress:
def __init__(self, progress: float = 0, message: str = None):
"""
@@ -7,14 +11,17 @@ def __init__(self, progress: float = 0, message: str = None):
"""
self._progress = progress
self._message = message
-
+ logger.setLevel(logging.INFO)
+
def set_status(self, progress: float = None, message: str = None):
if progress is not None:
self._progress = progress
if message is not None:
self._message = message
+ logger.info(f"Progress: {self._progress} Message: {self._message}")
+ return self
-
+
class JobProgressRunpod(JobProgress):
def __init__(self, runpod_job, progress: float = 0, message: str = None):
super().__init__(progress=progress, message=message)
diff --git a/apipod/engine/queue/job_queue.py b/apipod/engine/queue/job_queue.py
index 985d239..5aa6f0c 100644
--- a/apipod/engine/queue/job_queue.py
+++ b/apipod/engine/queue/job_queue.py
@@ -130,7 +130,7 @@ def _inject_job_progress(self, job: T) -> T:
job_progress_params = [
p for p in sig.parameters.values()
- if p.name == "job_progress" or "FastJobProgress" in str(p.annotation)
+ if p.name == "job_progress" or "JobProgress" in str(p.annotation)
]
for job_progress_param in job_progress_params:
job.job_params[job_progress_param.name] = job.job_progress
From 4cee9af5b4056fd203a092aed256a8bb2084baf3 Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Fri, 12 Jun 2026 10:39:16 +0200
Subject: [PATCH 08/38] added field values for descriptive pydantic schemas
---
apipod/common/schemas.py | 268 +++++++++---------
.../{llm => schema_extension}/__init__.py | 0
.../schema_mixin.py} | 2 +-
3 files changed, 135 insertions(+), 135 deletions(-)
rename apipod/engine/{llm => schema_extension}/__init__.py (100%)
rename apipod/engine/{llm/base_llm_mixin.py => schema_extension/schema_mixin.py} (99%)
diff --git a/apipod/common/schemas.py b/apipod/common/schemas.py
index d8216a2..a25417d 100644
--- a/apipod/common/schemas.py
+++ b/apipod/common/schemas.py
@@ -12,7 +12,7 @@
from types import UnionType
from typing import Any, List, Literal, Optional, Union, get_args, get_origin
-from pydantic import BaseModel, model_validator
+from pydantic import BaseModel, Field, model_validator
from media_toolkit import AudioFile, ImageFile, MediaFile, VideoFile, media_from_any
@@ -84,23 +84,23 @@ def _convert_media_strings(cls, data: Any) -> Any:
# =====================================================
class ChatMessage(APIPodSchemaBase):
- role: Literal["system", "user", "assistant"]
- content: str
+ role: Literal["system", "user", "assistant"] = Field(description="The role of the message author (system, user, or assistant).")
+ content: str = Field(description="The content of the message.")
class ChatCompletionRequest(APIPodSchemaBase):
- model: str
- messages: List[ChatMessage]
+ model: str = Field(description="ID of the model to use for this request.")
+ messages: List[ChatMessage] = Field(description="A list of messages comprising the conversation history.")
- temperature: float = 0.7
- max_tokens: Optional[int] = None
- top_p: float = 1.0
- n: int = 1
- stream: bool = False
- stop: Optional[Union[str, List[str]]] = None
- presence_penalty: float = 0.0
- frequency_penalty: float = 0.0
- user: Optional[str] = None
+ temperature: float = Field(default=0.7, description="Sampling temperature between 0.0 and 2.0. Higher values make output more random.")
+ max_tokens: Optional[int] = Field(default=None, description="The maximum number of tokens to generate in the completion.")
+ top_p: float = Field(default=1.0, description="Nucleus sampling threshold. Only tokens with cumulative probability >= top_p are considered.")
+ n: int = Field(default=1, description="How many completions to generate for each prompt.")
+ stream: bool = Field(default=False, description="If set, partial message deltas will be sent as server-sent events.")
+ stop: Optional[Union[str, List[str]]] = Field(default=None, description="Up to 4 sequences where the API will stop generating further tokens.")
+ presence_penalty: float = Field(default=0.0, description="Penalty for new tokens based on whether they appear in the text so far.")
+ frequency_penalty: float = Field(default=0.0, description="Penalty for new tokens based on their existing frequency in the text so far.")
+ user: Optional[str] = Field(default=None, description="A unique identifier representing your end-user.")
# =====================================================
@@ -108,15 +108,15 @@ class ChatCompletionRequest(APIPodSchemaBase):
# =====================================================
class CompletionRequest(APIPodSchemaBase):
- model: str
- prompt: Union[str, List[str]]
+ model: str = Field(description="ID of the model to use for this request.")
+ prompt: Union[str, List[str]] = Field(description="The prompt(s) to generate completions for.")
- temperature: float = 0.7
- max_tokens: int = 16
- top_p: float = 1.0
- n: int = 1
- stream: bool = False
- stop: Optional[Union[str, List[str]]] = None
+ temperature: float = Field(default=0.7, description="Sampling temperature between 0.0 and 2.0.")
+ max_tokens: int = Field(default=16, description="The maximum number of tokens to generate.")
+ top_p: float = Field(default=1.0, description="Nucleus sampling threshold.")
+ n: int = Field(default=1, description="How many completions to generate.")
+ stream: bool = Field(default=False, description="Whether to stream partial progress.")
+ stop: Optional[Union[str, List[str]]] = Field(default=None, description="Sequences where the API will stop generating.")
# =====================================================
@@ -124,9 +124,9 @@ class CompletionRequest(APIPodSchemaBase):
# =====================================================
class EmbeddingRequest(APIPodSchemaBase):
- model: str
- input: Union[str, List[str]]
- user: Optional[str] = None
+ model: str = Field(description="ID of the model to use for generating embeddings.")
+ input: Union[str, List[str]] = Field(description="The input text to embed.")
+ user: Optional[str] = Field(default=None, description="A unique identifier representing your end-user.")
# =====================================================
@@ -134,16 +134,16 @@ class EmbeddingRequest(APIPodSchemaBase):
# =====================================================
class ImageGenerationRequest(APIPodSchemaBase):
- model: str
- prompt: str
+ model: str = Field(description="ID of the model to use for image generation.")
+ prompt: str = Field(description="A text description of the desired image(s).")
- negative_prompt: Optional[str] = None
- image: Optional[ImageFile] = None
- mask: Optional[ImageFile] = None
- size: Optional[str] = None
- num_images: int = 1
- seed: Optional[int] = None
- steps: Optional[int] = None
+ negative_prompt: Optional[str] = Field(default=None, description="A text description of what to exclude from the generated image.")
+ image: Optional[ImageFile] = Field(default=None, description="An optional reference image for image-to-image or inpainting tasks.")
+ mask: Optional[ImageFile] = Field(default=None, description="An optional mask image for inpainting, where white pixels indicate areas to edit.")
+ size: Optional[str] = Field(default=None, description="The size of the generated images, e.g. 1024x1024.")
+ num_images: int = Field(default=1, description="The number of images to generate.")
+ seed: Optional[int] = Field(default=None, description="Random seed for reproducible generation.")
+ steps: Optional[int] = Field(default=None, description="The number of inference steps to perform.")
# =====================================================
@@ -151,14 +151,14 @@ class ImageGenerationRequest(APIPodSchemaBase):
# =====================================================
class VideoGenerationRequest(APIPodSchemaBase):
- model: str
- prompt: str
+ model: str = Field(description="ID of the model to use for video generation.")
+ prompt: str = Field(description="A text description of the desired video.")
- image: Optional[ImageFile] = None
- duration_s: float = 5.0
- fps: int = 24
- aspect_ratio: Optional[str] = None
- seed: Optional[int] = None
+ image: Optional[ImageFile] = Field(default=None, description="An optional reference image (frame0) to start the video from.")
+ duration_s: float = Field(default=5.0, description="The desired duration of the video in seconds.")
+ fps: int = Field(default=24, description="Frames per second for the generated video.")
+ aspect_ratio: Optional[str] = Field(default=None, description="The aspect ratio of the generated video, e.g. 16:9.")
+ seed: Optional[int] = Field(default=None, description="Random seed for reproducible generation.")
# =====================================================
@@ -166,14 +166,14 @@ class VideoGenerationRequest(APIPodSchemaBase):
# =====================================================
class AudioRequest(APIPodSchemaBase):
- model: str
+ model: str = Field(description="ID of the model to use for audio tasks (TTS, STT, or music).")
- text: Optional[str] = None
- audio: Optional[AudioFile] = None
- voice: Optional[str] = None
- language: Optional[str] = None
- format: Optional[str] = None
- duration_s: Optional[float] = None
+ text: Optional[str] = Field(default=None, description="The input text for text-to-speech or music generation.")
+ audio: Optional[AudioFile] = Field(default=None, description="The input audio file for speech-to-text tasks.")
+ voice: Optional[str] = Field(default=None, description="The voice ID or style to use for audio generation.")
+ language: Optional[str] = Field(default=None, description="The language of the input audio (for STT) or target language (for TTS).")
+ format: Optional[str] = Field(default=None, description="The desired output audio format, e.g. mp3, wav, flac.")
+ duration_s: Optional[float] = Field(default=None, description="The desired duration of the generated audio in seconds.")
# =====================================================
@@ -181,12 +181,12 @@ class AudioRequest(APIPodSchemaBase):
# =====================================================
class Generation3DRequest(APIPodSchemaBase):
- model: str
+ model: str = Field(description="ID of the model to use for 3D generation.")
- prompt: Optional[str] = None
- image: Optional[ImageFile] = None
- output_format: str = "glb"
- seed: Optional[int] = None
+ prompt: Optional[str] = Field(default=None, description="A text description of the 3D object to generate.")
+ image: Optional[ImageFile] = Field(default=None, description="An optional reference image to generate the 3D object from.")
+ output_format: str = Field(default="glb", description="The desired 3D output format (glb, obj, fbx, etc.).")
+ seed: Optional[int] = Field(default=None, description="Random seed for reproducible generation.")
# =====================================================
@@ -194,12 +194,12 @@ class Generation3DRequest(APIPodSchemaBase):
# =====================================================
class VisionRequest(APIPodSchemaBase):
- model: str
- image: ImageFile
+ model: str = Field(description="ID of the model to use for vision tasks (OCR, detection, etc.).")
+ image: ImageFile = Field(description="The image to process for vision tasks.")
- labels: Optional[List[str]] = None
- threshold: Optional[float] = None
- return_boxes: bool = False
+ labels: Optional[List[str]] = Field(default=None, description="Optional list of labels to look for (e.g. for object detection).")
+ threshold: Optional[float] = Field(default=None, description="Confidence threshold for detection or classification.")
+ return_boxes: bool = Field(default=False, description="Whether to return bounding box coordinates for detected objects.")
# =====================================================
@@ -207,12 +207,12 @@ class VisionRequest(APIPodSchemaBase):
# =====================================================
class MultimodalEmbeddingRequest(APIPodSchemaBase):
- model: str
+ model: str = Field(description="ID of the model to use for generating multimodal embeddings.")
- input: Optional[Union[str, List[str]]] = None
- image: Optional[ImageFile] = None
- audio: Optional[AudioFile] = None
- user: Optional[str] = None
+ input: Optional[Union[str, List[str]]] = Field(default=None, description="The input text to embed.")
+ image: Optional[ImageFile] = Field(default=None, description="The input image to embed.")
+ audio: Optional[AudioFile] = Field(default=None, description="The input audio to embed.")
+ user: Optional[str] = Field(default=None, description="A unique identifier representing your end-user.")
# Supported request schemas that should be interpreted as JSON bodies
@@ -235,9 +235,9 @@ class MultimodalEmbeddingRequest(APIPodSchemaBase):
# =====================================================
class Usage(APIPodSchemaBase):
- prompt_tokens: int
- completion_tokens: Optional[int] = None
- total_tokens: int
+ prompt_tokens: int = Field(description="Number of tokens in the prompt.")
+ completion_tokens: Optional[int] = Field(default=None, description="Number of tokens in the generated completion.")
+ total_tokens: int = Field(description="Total number of tokens used in the request (prompt + completion).")
# =====================================================
@@ -245,23 +245,23 @@ class Usage(APIPodSchemaBase):
# =====================================================
class ChatCompletionMessage(APIPodSchemaBase):
- role: Literal["assistant"]
- content: str
+ role: Literal["assistant"] = Field(description="The role of the message author, always 'assistant'.")
+ content: str = Field(description="The content of the message.")
class ChatCompletionChoice(APIPodSchemaBase):
- index: int
- message: ChatCompletionMessage
- finish_reason: Literal["stop", "length", "content_filter"]
+ index: int = Field(description="The index of the choice in the list of choices.")
+ message: ChatCompletionMessage = Field(description="A chat completion message generated by the model.")
+ finish_reason: Literal["stop", "length", "content_filter"] = Field(description="The reason the model stopped generating tokens.")
class ChatCompletionResponse(APIPodSchemaBase):
- id: str
- object: Literal["chat.completion"]
- created: int
- model: str
- choices: List[ChatCompletionChoice]
- usage: Usage
+ id: str = Field(description="Unique identifier for the chat completion.")
+ object: Literal["chat.completion"] = Field(description="The object type, always 'chat.completion'.")
+ created: int = Field(description="The Unix timestamp when the chat completion was created.")
+ model: str = Field(description="The model used for the chat completion.")
+ choices: List[ChatCompletionChoice] = Field(description="A list of chat completion choices.")
+ usage: Usage = Field(description="Usage statistics for the completion request.")
# =====================================================
@@ -306,19 +306,19 @@ class EmbeddingResponse(APIPodSchemaBase):
# =====================================================
class ImageGenerationData(APIPodSchemaBase):
- url: Optional[str] = None
- b64_json: Optional[str] = None
- revised_prompt: Optional[str] = None
- seed: Optional[int] = None
+ url: Optional[str] = Field(default=None, description="The URL of the generated image.")
+ b64_json: Optional[str] = Field(default=None, description="The base64-encoded JSON of the generated image.")
+ revised_prompt: Optional[str] = Field(default=None, description="The prompt as revised by the model, if applicable.")
+ seed: Optional[int] = Field(default=None, description="The seed used for generation.")
class ImageGenerationResponse(APIPodSchemaBase):
- id: str
- object: Literal["image_generation"]
- created: int
- model: str
- data: List[ImageGenerationData]
- usage: Optional[Usage] = None
+ id: str = Field(description="Unique identifier for the generation request.")
+ object: Literal["image_generation"] = Field(description="The object type, always 'image_generation'.")
+ created: int = Field(description="The Unix timestamp when the generation was created.")
+ model: str = Field(description="The model used for generation.")
+ data: List[ImageGenerationData] = Field(description="The generated image data.")
+ usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
# =====================================================
@@ -326,18 +326,18 @@ class ImageGenerationResponse(APIPodSchemaBase):
# =====================================================
class VideoGenerationData(APIPodSchemaBase):
- url: Optional[str] = None
- duration_s: Optional[float] = None
- seed: Optional[int] = None
+ url: Optional[str] = Field(default=None, description="The URL of the generated video.")
+ duration_s: Optional[float] = Field(default=None, description="The duration of the generated video in seconds.")
+ seed: Optional[int] = Field(default=None, description="The seed used for generation.")
class VideoGenerationResponse(APIPodSchemaBase):
- id: str
- object: Literal["video_generation"]
- created: int
- model: str
- data: List[VideoGenerationData]
- usage: Optional[Usage] = None
+ id: str = Field(description="Unique identifier for the generation request.")
+ object: Literal["video_generation"] = Field(description="The object type, always 'video_generation'.")
+ created: int = Field(description="The Unix timestamp when the generation was created.")
+ model: str = Field(description="The model used for generation.")
+ data: List[VideoGenerationData] = Field(description="The generated video data.")
+ usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
# =====================================================
@@ -345,19 +345,19 @@ class VideoGenerationResponse(APIPodSchemaBase):
# =====================================================
class AudioData(APIPodSchemaBase):
- audio: Optional[str] = None
- text: Optional[str] = None
- language: Optional[str] = None
- duration_s: Optional[float] = None
+ audio: Optional[str] = Field(default=None, description="The URL or base64 data of the generated audio.")
+ text: Optional[str] = Field(default=None, description="The transcribed text, if applicable.")
+ language: Optional[str] = Field(default=None, description="The detected language, if applicable.")
+ duration_s: Optional[float] = Field(default=None, description="The duration of the audio in seconds.")
class AudioResponse(APIPodSchemaBase):
- id: str
- object: Literal["audio"]
- created: int
- model: str
- data: List[AudioData]
- usage: Optional[Usage] = None
+ id: str = Field(description="Unique identifier for the request.")
+ object: Literal["audio"] = Field(description="The object type, always 'audio'.")
+ created: int = Field(description="The Unix timestamp when the response was created.")
+ model: str = Field(description="The model used for the task.")
+ data: List[AudioData] = Field(description="The resulting audio data.")
+ usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
# =====================================================
@@ -365,18 +365,18 @@ class AudioResponse(APIPodSchemaBase):
# =====================================================
class Generation3DData(APIPodSchemaBase):
- url: Optional[str] = None
- output_format: Optional[str] = None
- seed: Optional[int] = None
+ url: Optional[str] = Field(default=None, description="The URL of the generated 3D asset.")
+ output_format: Optional[str] = Field(default=None, description="The format of the generated 3D asset.")
+ seed: Optional[int] = Field(default=None, description="The seed used for generation.")
class Generation3DResponse(APIPodSchemaBase):
- id: str
- object: Literal["generation_3d"]
- created: int
- model: str
- data: List[Generation3DData]
- usage: Optional[Usage] = None
+ id: str = Field(description="Unique identifier for the request.")
+ object: Literal["generation_3d"] = Field(description="The object type, always 'generation_3d'.")
+ created: int = Field(description="The Unix timestamp when the response was created.")
+ model: str = Field(description="The model used for generation.")
+ data: List[Generation3DData] = Field(description="The generated 3D data.")
+ usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
# =====================================================
@@ -384,23 +384,23 @@ class Generation3DResponse(APIPodSchemaBase):
# =====================================================
class VisionLabel(APIPodSchemaBase):
- label: str
- score: float
- box: Optional[List[float]] = None
+ label: str = Field(description="The label of the detected object or classification.")
+ score: float = Field(description="The confidence score of the detection or classification.")
+ box: Optional[List[float]] = Field(default=None, description="The bounding box coordinates [ymin, xmin, ymax, xmax], if applicable.")
class VisionData(APIPodSchemaBase):
- labels: List[VisionLabel] = []
- text: Optional[str] = None
+ labels: List[VisionLabel] = Field(default_factory=list, description="A list of detected labels and scores.")
+ text: Optional[str] = Field(default=None, description="The extracted text from OCR, if applicable.")
class VisionResponse(APIPodSchemaBase):
- id: str
- object: Literal["vision"]
- created: int
- model: str
- data: List[VisionData]
- usage: Optional[Usage] = None
+ id: str = Field(description="Unique identifier for the request.")
+ object: Literal["vision"] = Field(description="The object type, always 'vision'.")
+ created: int = Field(description="The Unix timestamp when the response was created.")
+ model: str = Field(description="The model used for the vision task.")
+ data: List[VisionData] = Field(description="The resulting vision data.")
+ usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
# =====================================================
@@ -408,17 +408,17 @@ class VisionResponse(APIPodSchemaBase):
# =====================================================
class MultimodalEmbeddingData(APIPodSchemaBase):
- object: Literal["embedding"] = "embedding"
- embedding: List[float]
- index: int
- modality: Optional[Literal["text", "image", "audio"]] = None
+ object: Literal["embedding"] = Field(default="embedding", description="The object type, always 'embedding'.")
+ embedding: List[float] = Field(description="The embedding vector.")
+ index: int = Field(description="The index of the embedding in the list of inputs.")
+ modality: Optional[Literal["text", "image", "audio"]] = Field(default=None, description="The modality of the input that generated this embedding.")
class MultimodalEmbeddingResponse(APIPodSchemaBase):
- object: Literal["list"]
- data: List[MultimodalEmbeddingData]
- model: str
- usage: Optional[Usage] = None
+ object: Literal["list"] = Field(description="The object type, always 'list'.")
+ data: List[MultimodalEmbeddingData] = Field(description="The list of embedding data objects.")
+ model: str = Field(description="The model used for generating embeddings.")
+ usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
# ======================================================
diff --git a/apipod/engine/llm/__init__.py b/apipod/engine/schema_extension/__init__.py
similarity index 100%
rename from apipod/engine/llm/__init__.py
rename to apipod/engine/schema_extension/__init__.py
diff --git a/apipod/engine/llm/base_llm_mixin.py b/apipod/engine/schema_extension/schema_mixin.py
similarity index 99%
rename from apipod/engine/llm/base_llm_mixin.py
rename to apipod/engine/schema_extension/schema_mixin.py
index e27cd63..ae3c1f3 100644
--- a/apipod/engine/llm/base_llm_mixin.py
+++ b/apipod/engine/schema_extension/schema_mixin.py
@@ -16,7 +16,7 @@
)
-class _BaseLLMMixin:
+class _SchemaExtMixin:
"""
Mixin class for Base LLM functionality
"""
From e1961e0de4dd1ce3f3610bed94befb873fbb8626 Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Sun, 14 Jun 2026 11:26:27 +0200
Subject: [PATCH 09/38] Implemented standardized pydantic schema support. I
refactored the decoraters to support media-toolkit file parsing in nested
pydantic models. In schema endpoints the result is also wrapped into the
corresponding response format if the user lazy returns just for example an
AudioFile.
---
README.md | 53 ++-
apipod/common/schemas/__init__.py | 36 ++
apipod/common/schemas/media_files.py | 87 ++++
apipod/common/{ => schemas}/schemas.py | 303 +++++++------
.../backend/fastapi/file_handling_mixin.py | 3 +-
apipod/engine/backend/fastapi/llm_mixin.py | 29 --
apipod/engine/backend/fastapi/router.py | 257 +++++------
apipod/engine/backend/runpod/llm_mixin.py | 20 -
apipod/engine/backend/runpod/router.py | 100 ++---
apipod/engine/backend/schema_resolve.py | 235 +++++++++++
apipod/engine/endpoint_config.py | 22 +-
apipod/engine/files/base_file_mixin.py | 121 +++++-
apipod/engine/jobs/job_result.py | 61 +--
apipod/engine/schema_extension/__init__.py | 0
.../engine/schema_extension/schema_mixin.py | 166 --------
apipod/engine/signatures/policies.py | 35 +-
docs/README_TECH.md | 132 +++++-
examples/test_service_six_schemas.py | 38 +-
test/llm/test_llm_mock.py | 14 +-
test/llm/test_six_schemas.py | 399 ------------------
20 files changed, 980 insertions(+), 1131 deletions(-)
create mode 100644 apipod/common/schemas/__init__.py
create mode 100644 apipod/common/schemas/media_files.py
rename apipod/common/{ => schemas}/schemas.py (52%)
delete mode 100644 apipod/engine/backend/fastapi/llm_mixin.py
delete mode 100644 apipod/engine/backend/runpod/llm_mixin.py
create mode 100644 apipod/engine/backend/schema_resolve.py
delete mode 100644 apipod/engine/schema_extension/__init__.py
delete mode 100644 apipod/engine/schema_extension/schema_mixin.py
delete mode 100644 test/llm/test_six_schemas.py
diff --git a/README.md b/README.md
index 360aba9..6d2b1f4 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,52 @@ def transcribe(audio: AudioFile):
return {"transcription": "..."}
```
+### 🤖 Standardized AI schemas (OpenAI-compatible)
+
+APIPod ships request/response schemas for the common AI payloads: chat, completions, embeddings, image/video/3D generation, vision, transcription, text-to-speech and voice cloning. Annotate one parameter with a request schema and APIPod handles validation, media parsing, response wrapping and streaming. The wire format mirrors the OpenAI API, so OpenAI-compatible clients work out of the box. `model` is optional everywhere — your service usually *is* the model.
+
+**Chat** — return a plain string (or a dict / full response object) and get a complete `chat.completion` envelope. Return a generator and `"stream": true` requests receive server-sent events:
+
+```python
+from apipod import APIPod
+from apipod.common.schemas import ChatCompletionRequest
+
+app = APIPod()
+
+@app.endpoint("/chat")
+def chat(request: ChatCompletionRequest):
+ answer = my_llm.generate(request.messages, temperature=request.temperature)
+ return answer # auto-wrapped into a ChatCompletionResponse
+```
+
+**Text-to-speech** — return an `AudioFile`; clients get a JSON envelope with the audio as base64, or raw audio chunks when they send `"stream": true`:
+
+```python
+from apipod import AudioFile
+from apipod.common.schemas import SpeechRequest
+
+@app.endpoint("/audio/speech")
+def speak(request: SpeechRequest):
+ # request.voice is a named voice (str) or an already-parsed AudioFile for cloning
+ samples, rate = my_tts.synthesize(request.input, voice=request.voice)
+ return AudioFile().from_np_array(samples, sample_rate=rate, audio_format="wav")
+```
+
+**Image generation** — media fields inside schemas (e.g. `request.image` for img2img) arrive as parsed media-toolkit objects, no matter if the client sent an upload, URL or base64:
+
+```python
+from apipod import ImageFile
+from apipod.common.schemas import ImageGenerationRequest
+
+@app.endpoint("/images/generations")
+def generate(request: ImageGenerationRequest):
+ init_image = request.image.to_np_array() if request.image else None # ImageFile, ready to use
+ result = my_model.generate(request.prompt, image=init_image, seed=request.seed)
+ return ImageFile().from_np_array(result) # auto-wrapped into ImageGenerationResponse
+```
+
+Need extra parameters? Subclass a schema (`class MyRequest(SpeechRequest): style: str = "calm"`) — detection and wrapping keep working.
+
### ☁️ Serverless Routing
When deploying to serverless platforms like **RunPod**, standard web frameworks often fail because they lack the necessary routing logic for the platform's specific entry points. **APIPod** detects the environment and handles the routing automatically—no separate "handler" function required.
@@ -184,9 +230,9 @@ The configuration is controlled through a combination of:
| ---------------- | ------------ | ----------- | --------------------------------- |
| `socaity` | `dedicated` | `auto` | FastAPI |
| `socaity` | `dedicated` | `localhost` | FastAPI + job queue *(test mode)* |
-| `socaity` | `dedicated` | `runpod` | Celery backend *(planned)* |
-| `socaity` | `dedicated` | `scaleway` | Celery backend *(planned)* |
-| `socaity` | `dedicated` | `azure` | Celery backend *(planned)* |
+| `socaity` | `dedicated` | `runpod` | FastAPI + load balancer *(Coming soon)* |
+| `socaity` | `dedicated` | `scaleway` | FastAPI *(Coming soon)* |
+| `socaity` | `dedicated` | `azure` | FastAPI *(Coming soon)* |
| `socaity` | `serverless` | `auto` | RunPod router backend |
| `socaity` | `serverless` | `localhost` | FastAPI + job queue *(test mode)* |
| `socaity` | `serverless` | `runpod` | RunPod router backend |
@@ -299,7 +345,6 @@ result = task.get_result()
## Roadmap
- MCP protocol support.
-- OpenAI-compatible default endpoints for LLMs
- Improve async support.
---
diff --git a/apipod/common/schemas/__init__.py b/apipod/common/schemas/__init__.py
new file mode 100644
index 0000000..f33bc08
--- /dev/null
+++ b/apipod/common/schemas/__init__.py
@@ -0,0 +1,36 @@
+from .schemas import (
+ APIPodSchemaBase, Usage,
+ ChatMessage, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionChoice, ChatCompletionMessage,
+ CompletionRequest, CompletionResponse, CompletionChoice,
+ EmbeddingRequest, EmbeddingResponse, EmbeddingData,
+ ImageGenerationRequest, ImageGenerationResponse,
+ VideoGenerationRequest, VideoGenerationResponse,
+ TranscriptionRequest, TranscriptionResponse, TranscriptionSegment, TranscriptionWord,
+ SpeechRequest, SpeechResponse,
+ CreateVoiceRequest, VoiceResponse,
+ VoiceConversionRequest, VoiceConversionResponse,
+ Generation3DRequest, Generation3DResponse,
+ VisionRequest, VisionResponse, VisionData, VisionLabel,
+ MultimodalEmbeddingRequest, MultimodalEmbeddingResponse, MultimodalEmbeddingData,
+ ChatDelta, ChatStreamChoice, ChatCompletionChunk,
+)
+
+from .media_files import FileModel, ImageFileModel, AudioFileModel, VideoFileModel, ThreeDFileModel
+
+__all__ = [
+ "APIPodSchemaBase", "Usage",
+ "ChatMessage", "ChatCompletionRequest", "ChatCompletionResponse", "ChatCompletionChoice", "ChatCompletionMessage",
+ "CompletionRequest", "CompletionResponse", "CompletionChoice",
+ "EmbeddingRequest", "EmbeddingResponse", "EmbeddingData",
+ "ImageGenerationRequest", "ImageGenerationResponse",
+ "VideoGenerationRequest", "VideoGenerationResponse",
+ "TranscriptionRequest", "TranscriptionResponse", "TranscriptionSegment", "TranscriptionWord",
+ "SpeechRequest", "SpeechResponse",
+ "CreateVoiceRequest", "VoiceResponse",
+ "VoiceConversionRequest", "VoiceConversionResponse",
+ "Generation3DRequest", "Generation3DResponse",
+ "VisionRequest", "VisionResponse", "VisionData", "VisionLabel",
+ "MultimodalEmbeddingRequest", "MultimodalEmbeddingResponse", "MultimodalEmbeddingData",
+ "ChatDelta", "ChatStreamChoice", "ChatCompletionChunk",
+ "FileModel", "ImageFileModel", "AudioFileModel", "VideoFileModel", "ThreeDFileModel",
+]
diff --git a/apipod/common/schemas/media_files.py b/apipod/common/schemas/media_files.py
new file mode 100644
index 0000000..b7e64dd
--- /dev/null
+++ b/apipod/common/schemas/media_files.py
@@ -0,0 +1,87 @@
+from pydantic import BaseModel, AnyUrl, model_validator
+from typing import Union, Optional
+
+
+class FileModel(BaseModel):
+ """
+ Wire-format mirror of a media-toolkit file: name, content type and the
+ content itself as base64 or URL. Plain strings (URL / base64) and
+ media-toolkit file objects are coerced automatically so the model can be
+ used directly as a field type in request and response schemas.
+ """
+
+ file_name: str
+ content_type: str
+ content: Union[str, bytes, AnyUrl] # base64 encoded or url
+ max_size_mb: Optional[float] = 4000
+
+ @model_validator(mode="before")
+ @classmethod
+ def _coerce_input(cls, value):
+ if isinstance(value, (str, bytes)):
+ return {"file_name": "file", "content_type": "application/octet-stream", "content": value}
+ if isinstance(value, FileModel):
+ # Re-validate a (sibling/parent) FileModel against the concrete field type.
+ return value if isinstance(value, cls) else value.model_dump()
+ if not isinstance(value, (dict, BaseModel)) and hasattr(value, "to_json"):
+ # media-toolkit files serialize to {file_name, content_type, content}.
+ return value.to_json()
+ return value
+
+ class Config:
+ json_schema_extra = {
+ "x-media-type": "MediaFile",
+ "example": {
+ "file_name": "example.csv",
+ "content_type": "text/csv",
+ "content": "https://example.com/example.csv",
+ }
+ }
+
+
+class ImageFileModel(FileModel):
+ class Config:
+ json_schema_extra = {
+ "x-media-type": "ImageFile",
+ "example": {
+ "file_name": "example.png",
+ "content_type": "image/png",
+ "content": "base64 encoded image data",
+ }
+ }
+
+
+class AudioFileModel(FileModel):
+ class Config:
+ json_schema_extra = {
+ "x-media-type": "AudioFile",
+ "example": {
+ "file_name": "example.mp3",
+ "content_type": "audio/mpeg",
+ "content": "base64 encoded audio data",
+ }
+ }
+
+
+class VideoFileModel(FileModel):
+ class Config:
+ json_schema_extra = {
+ "x-media-type": "VideoFile",
+ "example": {
+ "file_name": "example.mp4",
+ "content_type": "video/mp4",
+ "content": "base64 encoded video data",
+ }
+ }
+
+
+class ThreeDFileModel(FileModel):
+ class Config:
+ json_schema_extra = {
+ "x-media-type": "3DFile",
+ "example": {
+ "file_name": "example.glb",
+ "content_type": "model/gltf-binary",
+ "content": "base64 encoded 3D model data",
+ }
+ }
diff --git a/apipod/common/schemas.py b/apipod/common/schemas/schemas.py
similarity index 52%
rename from apipod/common/schemas.py
rename to apipod/common/schemas/schemas.py
index a25417d..bd9704d 100644
--- a/apipod/common/schemas.py
+++ b/apipod/common/schemas/schemas.py
@@ -8,26 +8,15 @@
models. Any provider (Flux, Stable Diffusion, ElevenLabs, Whisper, Suno,
DeepSeek, etc.) plugs into the same schemas and the routing layer
dispatches to whatever runs behind it.
-"""
-
-from types import UnionType
-from typing import Any, List, Literal, Optional, Union, get_args, get_origin
-from pydantic import BaseModel, Field, model_validator
-from media_toolkit import AudioFile, ImageFile, MediaFile, VideoFile, media_from_any
-
-_MEDIA_FIELD_TYPES = (ImageFile, AudioFile, VideoFile, MediaFile)
+`model` is optional on every request: an APIPod service typically serves
+exactly one model, so forcing clients to repeat its name adds nothing.
+"""
+from typing import Any, List, Literal, Optional, Union
+from pydantic import BaseModel, Field
-def _media_field_type(annotation: Any) -> Optional[type]:
- """Media class if annotation is media-typed (bare, Optional, or Union), else None."""
- if annotation in _MEDIA_FIELD_TYPES:
- return annotation
- if get_origin(annotation) in (Union, UnionType):
- for arg in get_args(annotation):
- if arg in _MEDIA_FIELD_TYPES:
- return arg
- return None
+from .media_files import FileModel, ImageFileModel, AudioFileModel, VideoFileModel, ThreeDFileModel
# =====================================================
@@ -42,55 +31,34 @@ class APIPodSchemaBase(BaseModel):
out of the box, but the schemas are provider-agnostic: anything that
matches the shape (Flux, SD, ElevenLabs, Whisper, in-house models...)
is a valid backend.
+
+ Media fields are declared with the FileModel variants (ImageFileModel,
+ AudioFileModel, ...) which accept uploads, FileModel JSON objects, URLs
+ and base64 strings. At runtime the file-handling layer replaces them
+ with parsed media-toolkit objects before the endpoint function runs.
"""
model_config = {
"extra": "forbid",
"validate_assignment": True,
"populate_by_name": True,
- # ImageFile / AudioFile / VideoFile / MediaFile are non-BaseModel types
- # used as field annotations; the pre-validator below converts URL /
- # base64 strings into instances before per-field validation.
- "arbitrary_types_allowed": True,
+ "arbitrary_types_allowed": True
}
- @model_validator(mode="before")
- @classmethod
- def _convert_media_strings(cls, data: Any) -> Any:
- """URL / data URI through media_from_any, bare base64 through from_base64."""
- if not isinstance(data, dict):
- return data
- for field_name, field_info in cls.model_fields.items():
- if field_name not in data:
- continue
- value = data[field_name]
- if not isinstance(value, str):
- continue
- media_type = _media_field_type(field_info.annotation)
- if media_type is None:
- continue
- data[field_name] = media_from_any(
- data=value,
- type_hint=media_type,
- use_temp_file=True,
- temp_dir=None,
- allow_reads_from_disk=False,
- )
- return data
-
# =====================================================
# Chat Completions - Input schemas
# =====================================================
+
class ChatMessage(APIPodSchemaBase):
role: Literal["system", "user", "assistant"] = Field(description="The role of the message author (system, user, or assistant).")
content: str = Field(description="The content of the message.")
class ChatCompletionRequest(APIPodSchemaBase):
- model: str = Field(description="ID of the model to use for this request.")
messages: List[ChatMessage] = Field(description="A list of messages comprising the conversation history.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
temperature: float = Field(default=0.7, description="Sampling temperature between 0.0 and 2.0. Higher values make output more random.")
max_tokens: Optional[int] = Field(default=None, description="The maximum number of tokens to generate in the completion.")
@@ -108,8 +76,8 @@ class ChatCompletionRequest(APIPodSchemaBase):
# =====================================================
class CompletionRequest(APIPodSchemaBase):
- model: str = Field(description="ID of the model to use for this request.")
prompt: Union[str, List[str]] = Field(description="The prompt(s) to generate completions for.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
temperature: float = Field(default=0.7, description="Sampling temperature between 0.0 and 2.0.")
max_tokens: int = Field(default=16, description="The maximum number of tokens to generate.")
@@ -124,8 +92,8 @@ class CompletionRequest(APIPodSchemaBase):
# =====================================================
class EmbeddingRequest(APIPodSchemaBase):
- model: str = Field(description="ID of the model to use for generating embeddings.")
input: Union[str, List[str]] = Field(description="The input text to embed.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
user: Optional[str] = Field(default=None, description="A unique identifier representing your end-user.")
@@ -134,12 +102,12 @@ class EmbeddingRequest(APIPodSchemaBase):
# =====================================================
class ImageGenerationRequest(APIPodSchemaBase):
- model: str = Field(description="ID of the model to use for image generation.")
prompt: str = Field(description="A text description of the desired image(s).")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
negative_prompt: Optional[str] = Field(default=None, description="A text description of what to exclude from the generated image.")
- image: Optional[ImageFile] = Field(default=None, description="An optional reference image for image-to-image or inpainting tasks.")
- mask: Optional[ImageFile] = Field(default=None, description="An optional mask image for inpainting, where white pixels indicate areas to edit.")
+ image: Optional[ImageFileModel] = Field(default=None, description="An optional reference image for image-to-image or inpainting tasks.")
+ mask: Optional[ImageFileModel] = Field(default=None, description="An optional mask image for inpainting, where white pixels indicate areas to edit.")
size: Optional[str] = Field(default=None, description="The size of the generated images, e.g. 1024x1024.")
num_images: int = Field(default=1, description="The number of images to generate.")
seed: Optional[int] = Field(default=None, description="Random seed for reproducible generation.")
@@ -151,29 +119,61 @@ class ImageGenerationRequest(APIPodSchemaBase):
# =====================================================
class VideoGenerationRequest(APIPodSchemaBase):
- model: str = Field(description="ID of the model to use for video generation.")
prompt: str = Field(description="A text description of the desired video.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
- image: Optional[ImageFile] = Field(default=None, description="An optional reference image (frame0) to start the video from.")
+ image: Optional[ImageFileModel] = Field(default=None, description="An optional reference image (frame0) to start the video from.")
duration_s: float = Field(default=5.0, description="The desired duration of the video in seconds.")
fps: int = Field(default=24, description="Frames per second for the generated video.")
aspect_ratio: Optional[str] = Field(default=None, description="The aspect ratio of the generated video, e.g. 16:9.")
seed: Optional[int] = Field(default=None, description="Random seed for reproducible generation.")
+ stream: bool = Field(default=False, description="If set, the generated video is streamed back as raw bytes instead of a JSON response.")
# =====================================================
-# Audio - Input schemas (TTS, STT, music)
+# Audio - Input schemas (transcription, speech, voices)
# =====================================================
-class AudioRequest(APIPodSchemaBase):
- model: str = Field(description="ID of the model to use for audio tasks (TTS, STT, or music).")
+class TranscriptionRequest(APIPodSchemaBase):
+ """Speech-to-text. Mirrors OpenAI POST /audio/transcriptions ('file' is called 'audio' here)."""
+
+ audio: AudioFileModel = Field(description="The audio file to transcribe. Accepts uploads, FileModel objects, URLs or base64.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: only specify if you serve multiple models.")
+
+ language: Optional[str] = Field(default=None, description="The language of the input audio in ISO-639-1 format (e.g. 'en'). Improves accuracy and latency.")
+ prompt: Optional[str] = Field(default=None, description="Optional text to guide the model's style or to continue a previous audio segment.")
+ response_format: Literal["json", "verbose_json"] = Field(default="json", description="Format of the transcript output.")
+ timestamp_granularities: Optional[List[Literal["word", "segment"]]] = Field(default=None, description="Timestamp detail to include; requires response_format='verbose_json'.")
+ stream: bool = Field(default=False, description="If set, partial transcript deltas will be sent as server-sent events.")
+
+
+class SpeechRequest(APIPodSchemaBase):
+ """Text-to-speech. Mirrors OpenAI POST /audio/speech."""
+
+ input: str = Field(description="The text to generate audio for.")
+ voice: Optional[Union[str, AudioFileModel]] = Field(default=None, description="A named voice of the service OR a reference audio / voice embedding file for voice cloning.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
+
+ instructions: Optional[str] = Field(default=None, description="Additional instructions to control voice, emotion or style.")
+ response_format: Literal["mp3", "wav", "opus", "flac", "pcm"] = Field(default="mp3", description="The desired output audio format.")
+ speed: float = Field(default=1.0, description="Playback speed of the generated audio (0.25 to 4.0).")
+ stream: bool = Field(default=False, description="If set, the generated audio is streamed back as raw audio chunks instead of a JSON response.")
- text: Optional[str] = Field(default=None, description="The input text for text-to-speech or music generation.")
- audio: Optional[AudioFile] = Field(default=None, description="The input audio file for speech-to-text tasks.")
- voice: Optional[str] = Field(default=None, description="The voice ID or style to use for audio generation.")
- language: Optional[str] = Field(default=None, description="The language of the input audio (for STT) or target language (for TTS).")
- format: Optional[str] = Field(default=None, description="The desired output audio format, e.g. mp3, wav, flac.")
- duration_s: Optional[float] = Field(default=None, description="The desired duration of the generated audio in seconds.")
+
+class CreateVoiceRequest(APIPodSchemaBase):
+ """Voice cloning: create a reusable voice (embedding) from an audio sample. Mirrors OpenAI POST /audio/voices."""
+
+ name: str = Field(description="A name for the new voice.")
+ audio_sample: AudioFileModel = Field(description="A clean speech sample (~5-20s) of the voice to clone.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
+
+
+class VoiceConversionRequest(APIPodSchemaBase):
+ """Voice-to-voice: re-render existing audio with another voice (named, sample or embedding)."""
+
+ audio: AudioFileModel = Field(description="The audio whose content should be kept.")
+ voice: Union[str, AudioFileModel] = Field(description="The target voice: a named voice of the service OR a reference audio / voice embedding file.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
# =====================================================
@@ -181,10 +181,10 @@ class AudioRequest(APIPodSchemaBase):
# =====================================================
class Generation3DRequest(APIPodSchemaBase):
- model: str = Field(description="ID of the model to use for 3D generation.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
prompt: Optional[str] = Field(default=None, description="A text description of the 3D object to generate.")
- image: Optional[ImageFile] = Field(default=None, description="An optional reference image to generate the 3D object from.")
+ image: Optional[ImageFileModel] = Field(default=None, description="An optional reference image to generate the 3D object from.")
output_format: str = Field(default="glb", description="The desired 3D output format (glb, obj, fbx, etc.).")
seed: Optional[int] = Field(default=None, description="Random seed for reproducible generation.")
@@ -194,8 +194,8 @@ class Generation3DRequest(APIPodSchemaBase):
# =====================================================
class VisionRequest(APIPodSchemaBase):
- model: str = Field(description="ID of the model to use for vision tasks (OCR, detection, etc.).")
- image: ImageFile = Field(description="The image to process for vision tasks.")
+ image: ImageFileModel = Field(description="The image to process for vision tasks.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
labels: Optional[List[str]] = Field(default=None, description="Optional list of labels to look for (e.g. for object detection).")
threshold: Optional[float] = Field(default=None, description="Confidence threshold for detection or classification.")
@@ -207,29 +207,14 @@ class VisionRequest(APIPodSchemaBase):
# =====================================================
class MultimodalEmbeddingRequest(APIPodSchemaBase):
- model: str = Field(description="ID of the model to use for generating multimodal embeddings.")
+ model: Optional[str] = Field(default=None, description="ID of the model to use. Optional: an APIPod service usually serves exactly one model.")
input: Optional[Union[str, List[str]]] = Field(default=None, description="The input text to embed.")
- image: Optional[ImageFile] = Field(default=None, description="The input image to embed.")
- audio: Optional[AudioFile] = Field(default=None, description="The input audio to embed.")
+ image: Optional[ImageFileModel] = Field(default=None, description="The input image to embed.")
+ audio: Optional[AudioFileModel] = Field(default=None, description="The input audio to embed.")
user: Optional[str] = Field(default=None, description="A unique identifier representing your end-user.")
-# Supported request schemas that should be interpreted as JSON bodies
-# by router decorators, even when endpoint authors do not specify Body(...).
-SUPPORTED_LLM_REQUEST_SCHEMAS = (
- ChatCompletionRequest,
- CompletionRequest,
- EmbeddingRequest,
- ImageGenerationRequest,
- VideoGenerationRequest,
- AudioRequest,
- Generation3DRequest,
- VisionRequest,
- MultimodalEmbeddingRequest,
-)
-
-
# =====================================================
# Shared output schemas
# =====================================================
@@ -245,7 +230,7 @@ class Usage(APIPodSchemaBase):
# =====================================================
class ChatCompletionMessage(APIPodSchemaBase):
- role: Literal["assistant"] = Field(description="The role of the message author, always 'assistant'.")
+ role: Literal["assistant"] = Field(default="assistant", description="The role of the message author, always 'assistant'.")
content: str = Field(description="The content of the message.")
@@ -256,12 +241,11 @@ class ChatCompletionChoice(APIPodSchemaBase):
class ChatCompletionResponse(APIPodSchemaBase):
- id: str = Field(description="Unique identifier for the chat completion.")
- object: Literal["chat.completion"] = Field(description="The object type, always 'chat.completion'.")
+ object: Literal["chat.completion"] = Field(default="chat.completion", description="The object type, always 'chat.completion'.")
created: int = Field(description="The Unix timestamp when the chat completion was created.")
- model: str = Field(description="The model used for the chat completion.")
+ model: Optional[str] = Field(default=None, description="The model used for the chat completion.")
choices: List[ChatCompletionChoice] = Field(description="A list of chat completion choices.")
- usage: Usage = Field(description="Usage statistics for the completion request.")
+ usage: Optional[Usage] = Field(default=None, description="Usage statistics for the completion request.")
# =====================================================
@@ -269,19 +253,18 @@ class ChatCompletionResponse(APIPodSchemaBase):
# =====================================================
class CompletionChoice(APIPodSchemaBase):
- text: str
- index: int
+ text: str = Field(description="The generated completion text.")
+ index: int = Field(description="The index of the choice in the list of choices.")
logprobs: None = None
- finish_reason: Literal["stop", "length", "content_filter"]
+ finish_reason: Literal["stop", "length", "content_filter"] = Field(description="The reason the model stopped generating tokens.")
class CompletionResponse(APIPodSchemaBase):
- id: str
- object: Literal["text_completion"]
- created: int
- model: str
- choices: List[CompletionChoice]
- usage: Usage
+ object: Literal["text_completion"] = Field(default="text_completion", description="The object type, always 'text_completion'.")
+ created: int = Field(description="The Unix timestamp when the completion was created.")
+ model: Optional[str] = Field(default=None, description="The model used for the completion.")
+ choices: List[CompletionChoice] = Field(description="A list of completion choices.")
+ usage: Optional[Usage] = Field(default=None, description="Usage statistics for the completion request.")
# =====================================================
@@ -289,35 +272,27 @@ class CompletionResponse(APIPodSchemaBase):
# =====================================================
class EmbeddingData(APIPodSchemaBase):
- object: Literal["embedding"]
- embedding: List[float]
- index: int
+ object: Literal["embedding"] = Field(default="embedding", description="The object type, always 'embedding'.")
+ embedding: List[float] = Field(description="The embedding vector.")
+ index: int = Field(description="The index of the embedding in the list of inputs.")
class EmbeddingResponse(APIPodSchemaBase):
- object: Literal["list"]
- data: List[EmbeddingData]
- model: str
- usage: Usage
+ object: Literal["list"] = Field(default="list", description="The object type, always 'list'.")
+ data: List[EmbeddingData] = Field(description="The list of embedding data objects.")
+ model: Optional[str] = Field(default=None, description="The model used for generating embeddings.")
+ usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
# =====================================================
# Image Generation - Output schemas
# =====================================================
-class ImageGenerationData(APIPodSchemaBase):
- url: Optional[str] = Field(default=None, description="The URL of the generated image.")
- b64_json: Optional[str] = Field(default=None, description="The base64-encoded JSON of the generated image.")
- revised_prompt: Optional[str] = Field(default=None, description="The prompt as revised by the model, if applicable.")
- seed: Optional[int] = Field(default=None, description="The seed used for generation.")
-
-
class ImageGenerationResponse(APIPodSchemaBase):
- id: str = Field(description="Unique identifier for the generation request.")
- object: Literal["image_generation"] = Field(description="The object type, always 'image_generation'.")
+ object: Literal["image_generation"] = Field(default="image_generation", description="The object type, always 'image_generation'.")
created: int = Field(description="The Unix timestamp when the generation was created.")
- model: str = Field(description="The model used for generation.")
- data: List[ImageGenerationData] = Field(description="The generated image data.")
+ model: Optional[str] = Field(default=None, description="The model used for generation.")
+ data: List[ImageFileModel] = Field(description="The generated image data.")
usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
@@ -325,57 +300,76 @@ class ImageGenerationResponse(APIPodSchemaBase):
# Video Generation - Output schemas
# =====================================================
-class VideoGenerationData(APIPodSchemaBase):
- url: Optional[str] = Field(default=None, description="The URL of the generated video.")
- duration_s: Optional[float] = Field(default=None, description="The duration of the generated video in seconds.")
- seed: Optional[int] = Field(default=None, description="The seed used for generation.")
-
-
class VideoGenerationResponse(APIPodSchemaBase):
- id: str = Field(description="Unique identifier for the generation request.")
- object: Literal["video_generation"] = Field(description="The object type, always 'video_generation'.")
+ object: Literal["video_generation"] = Field(default="video_generation", description="The object type, always 'video_generation'.")
created: int = Field(description="The Unix timestamp when the generation was created.")
- model: str = Field(description="The model used for generation.")
- data: List[VideoGenerationData] = Field(description="The generated video data.")
+ model: Optional[str] = Field(default=None, description="The model used for generation.")
+ data: List[VideoFileModel] = Field(description="The generated video data.")
usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
# =====================================================
-# Audio - Output schemas
+# Audio - Output schemas (transcription, speech, voices)
# =====================================================
-class AudioData(APIPodSchemaBase):
- audio: Optional[str] = Field(default=None, description="The URL or base64 data of the generated audio.")
- text: Optional[str] = Field(default=None, description="The transcribed text, if applicable.")
- language: Optional[str] = Field(default=None, description="The detected language, if applicable.")
- duration_s: Optional[float] = Field(default=None, description="The duration of the audio in seconds.")
+class TranscriptionWord(APIPodSchemaBase):
+ word: str = Field(description="The transcribed word.")
+ start: float = Field(description="Start time of the word in seconds.")
+ end: float = Field(description="End time of the word in seconds.")
+
+
+class TranscriptionSegment(APIPodSchemaBase):
+ id: int = Field(default=0, description="Index of the segment.")
+ start: float = Field(description="Start time of the segment in seconds.")
+ end: float = Field(description="End time of the segment in seconds.")
+ text: str = Field(description="The transcribed text of the segment.")
+
+
+class TranscriptionResponse(APIPodSchemaBase):
+ """Mirrors the OpenAI transcription object: plain `text` plus optional verbose details."""
+ text: str = Field(description="The transcribed text.")
+ language: Optional[str] = Field(default=None, description="The detected or requested language of the audio.")
+ duration: Optional[float] = Field(default=None, description="Duration of the input audio in seconds.")
+ segments: Optional[List[TranscriptionSegment]] = Field(default=None, description="Segment-level details (verbose_json).")
+ words: Optional[List[TranscriptionWord]] = Field(default=None, description="Word-level timestamps (verbose_json).")
+ usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
-class AudioResponse(APIPodSchemaBase):
- id: str = Field(description="Unique identifier for the request.")
- object: Literal["audio"] = Field(description="The object type, always 'audio'.")
+class AudioEnvelopeBase(APIPodSchemaBase):
+ """Shared envelope for endpoints that return audio files (speech synthesis, voice conversion)."""
created: int = Field(description="The Unix timestamp when the response was created.")
- model: str = Field(description="The model used for the task.")
- data: List[AudioData] = Field(description="The resulting audio data.")
+ model: Optional[str] = Field(default=None, description="The model used for the task.")
+ data: List[AudioFileModel] = Field(description="The resulting audio file(s).")
usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
+class SpeechResponse(AudioEnvelopeBase):
+ object: Literal["audio.speech"] = Field(default="audio.speech", description="The object type, always 'audio.speech'.")
+
+
+class VoiceConversionResponse(AudioEnvelopeBase):
+ object: Literal["audio.conversion"] = Field(default="audio.conversion", description="The object type, always 'audio.conversion'.")
+
+
+class VoiceResponse(APIPodSchemaBase):
+ """A created (cloned) voice, optionally carrying its embedding file (e.g. a SpeechCraft .npz)."""
+ id: str = Field(description="Unique identifier of the voice.")
+ object: Literal["audio.voice"] = Field(default="audio.voice", description="The object type, always 'audio.voice'.")
+ name: str = Field(description="The name of the voice.")
+ created: int = Field(description="The Unix timestamp when the voice was created.")
+ model: Optional[str] = Field(default=None, description="The model used to create the voice.")
+ embedding: Optional[FileModel] = Field(default=None, description="The voice embedding file, if the service exposes it for reuse.")
+
+
# =====================================================
# 3D Generation - Output schemas
# =====================================================
-class Generation3DData(APIPodSchemaBase):
- url: Optional[str] = Field(default=None, description="The URL of the generated 3D asset.")
- output_format: Optional[str] = Field(default=None, description="The format of the generated 3D asset.")
- seed: Optional[int] = Field(default=None, description="The seed used for generation.")
-
-
class Generation3DResponse(APIPodSchemaBase):
- id: str = Field(description="Unique identifier for the request.")
- object: Literal["generation_3d"] = Field(description="The object type, always 'generation_3d'.")
+ object: Literal["generation_3d"] = Field(default="generation_3d", description="The object type, always 'generation_3d'.")
created: int = Field(description="The Unix timestamp when the response was created.")
- model: str = Field(description="The model used for generation.")
- data: List[Generation3DData] = Field(description="The generated 3D data.")
+ model: Optional[str] = Field(default=None, description="The model used for generation.")
+ data: List[ThreeDFileModel] = Field(description="The generated 3D data.")
usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
@@ -395,10 +389,9 @@ class VisionData(APIPodSchemaBase):
class VisionResponse(APIPodSchemaBase):
- id: str = Field(description="Unique identifier for the request.")
- object: Literal["vision"] = Field(description="The object type, always 'vision'.")
+ object: Literal["vision"] = Field(default="vision", description="The object type, always 'vision'.")
created: int = Field(description="The Unix timestamp when the response was created.")
- model: str = Field(description="The model used for the vision task.")
+ model: Optional[str] = Field(default=None, description="The model used for the vision task.")
data: List[VisionData] = Field(description="The resulting vision data.")
usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
@@ -415,9 +408,9 @@ class MultimodalEmbeddingData(APIPodSchemaBase):
class MultimodalEmbeddingResponse(APIPodSchemaBase):
- object: Literal["list"] = Field(description="The object type, always 'list'.")
+ object: Literal["list"] = Field(default="list", description="The object type, always 'list'.")
data: List[MultimodalEmbeddingData] = Field(description="The list of embedding data objects.")
- model: str = Field(description="The model used for generating embeddings.")
+ model: Optional[str] = Field(default=None, description="The model used for generating embeddings.")
usage: Optional[Usage] = Field(default=None, description="Token usage information, if applicable.")
@@ -427,14 +420,16 @@ class MultimodalEmbeddingResponse(APIPodSchemaBase):
class ChatDelta(BaseModel):
content: Optional[str] = None
+
class ChatStreamChoice(BaseModel):
index: int
delta: ChatDelta
finish_reason: Optional[str] = None
+
class ChatCompletionChunk(BaseModel):
id: str
object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
created: int
- model: str
- choices: List[ChatStreamChoice]
\ No newline at end of file
+ model: Optional[str] = None
+ choices: List[ChatStreamChoice]
diff --git a/apipod/engine/backend/fastapi/file_handling_mixin.py b/apipod/engine/backend/fastapi/file_handling_mixin.py
index fb74aba..f782cab 100644
--- a/apipod/engine/backend/fastapi/file_handling_mixin.py
+++ b/apipod/engine/backend/fastapi/file_handling_mixin.py
@@ -3,9 +3,10 @@
from typing import Any, Union, get_args, get_origin, Callable, List, Dict
from apipod.engine.backend.fastapi.LimitedUploadFile import LimitedUploadFile
from apipod.engine.signatures.upload import is_param_media_toolkit_file
-from apipod.engine.jobs.job_result import FileModel, ImageFileModel, AudioFileModel, VideoFileModel
+from apipod.common.schemas.media_files import FileModel, ImageFileModel, AudioFileModel, VideoFileModel
from apipod.engine.signatures.policies import FastAPISignaturePolicies
from apipod.engine.files.base_file_mixin import _BaseFileHandlingMixin
+from apipod.engine.endpoint_config import EndpointExecutionPlan
from apipod.engine.utils import replace_func_signature
from media_toolkit import MediaList, MediaDict, ImageFile, AudioFile, VideoFile, MediaFile
import functools
diff --git a/apipod/engine/backend/fastapi/llm_mixin.py b/apipod/engine/backend/fastapi/llm_mixin.py
deleted file mode 100644
index 41bb5e3..0000000
--- a/apipod/engine/backend/fastapi/llm_mixin.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from fastapi.responses import StreamingResponse
-
-from apipod.engine.llm.base_llm_mixin import _BaseLLMMixin
-
-
-class _FastApiLlmMixin(_BaseLLMMixin):
- """
- LLM execution logic for the FastAPI provider backend.
- """
-
- async def handle_llm_request(self, func, openai_req, should_use_queue, res_model, endpoint_type, **kwargs):
- should_stream = getattr(openai_req, "stream", False) and endpoint_type != "embedding"
-
- if should_stream:
- result = await self._execute_func(func, payload=openai_req, **kwargs)
- return StreamingResponse(self._stream_generator(result), media_type="text/event-stream")
-
- if should_use_queue:
- from apipod.engine.jobs.job_result import JobResultFactory
-
- job = self.job_queue.add_job(
- job_function=func,
- job_params={"payload": openai_req.dict()}
- )
- ret_job = JobResultFactory.from_base_job(job)
- return ret_job
-
- raw_res = await self._execute_func(func, payload=openai_req, **kwargs)
- return self._wrap_llm_response(raw_res, res_model, endpoint_type, openai_req)
diff --git a/apipod/engine/backend/fastapi/router.py b/apipod/engine/backend/fastapi/router.py
index 0c43256..5a5745a 100644
--- a/apipod/engine/backend/fastapi/router.py
+++ b/apipod/engine/backend/fastapi/router.py
@@ -3,7 +3,8 @@
import threading
import logging
from contextlib import asynccontextmanager
-from typing import Union, Callable, get_type_hints, Generator, AsyncGenerator, Iterator, AsyncIterator
+from collections.abc import AsyncIterator, Iterator
+from typing import Any, Union, Callable, get_origin, get_type_hints
from fastapi import APIRouter, FastAPI, Request, Response, status
from fastapi.exceptions import HTTPException
from fastapi.responses import JSONResponse, StreamingResponse
@@ -17,10 +18,16 @@
from apipod.engine.backend.fastapi.file_handling_mixin import _fast_api_file_handling_mixin
from apipod.engine.utils import normalize_name
from apipod.engine.backend.fastapi.exception_handling import _FastAPIExceptionHandler
-from apipod.engine.backend.fastapi.llm_mixin import _FastApiLlmMixin
+from apipod.engine.backend.schema_resolve import (
+ SchemaBinding,
+ SSE_STREAM_TAGS,
+ iter_media_chunks,
+ wrap_schema_response,
+)
+from media_toolkit import AudioFile, VideoFile
-class SocaityFastAPIRouter(APIRouter, _BaseBackend, _QueueMixin, _fast_api_file_handling_mixin, _FastAPIExceptionHandler, _FastApiLlmMixin):
+class SocaityFastAPIRouter(APIRouter, _BaseBackend, _QueueMixin, _fast_api_file_handling_mixin, _FastAPIExceptionHandler):
"""
FastAPI router extension that adds support for task endpoints.
@@ -69,7 +76,6 @@ def __init__(
_BaseBackend.__init__(self, title=title, summary=summary, *args, **kwargs)
_QueueMixin.__init__(self, job_queue=job_queue, *args, **kwargs)
_fast_api_file_handling_mixin.__init__(self, max_upload_file_size_mb=max_upload_file_size_mb, *args, **kwargs)
- _FastApiLlmMixin.__init__(self)
self.status = SERVER_HEALTH.INITIALIZING
@@ -296,12 +302,11 @@ async def _event_generator():
},
)
-
def endpoint(self, path: str, methods: list[str] | None = None, max_upload_file_size_mb: int = None, queue_size: int = 500, use_queue: bool = None, *args, **kwargs):
"""
Unified endpoint decorator.
- If LLM configuration is detected on the function, it creates an LLM endpoint.
+ If a parameter is annotated with a standardized request schema, it creates a schema endpoint.
If job_queue is configured (and use_queue is not False), it creates a task endpoint.
Otherwise, it creates a standard FastAPI endpoint.
@@ -328,33 +333,12 @@ def decorator(func: Callable) -> Callable:
route_kwargs=kwargs
)
- if plan.is_llm:
- return self._register_llm_endpoint(func, plan)
if plan.is_streaming:
- return self._create_streaming_endpoint_decorator(
- path=plan.path,
- methods=plan.methods,
- max_upload_file_size_mb=plan.max_upload_file_size_mb,
- args=plan.route_args,
- kwargs=plan.route_kwargs
- )(func)
+ return self._create_streaming_endpoint_decorator(plan)(func)
if plan.should_use_queue:
- return self._create_task_endpoint_decorator(
- path=plan.path,
- methods=plan.methods,
- max_upload_file_size_mb=plan.max_upload_file_size_mb,
- queue_size=plan.queue_size,
- args=plan.route_args,
- kwargs=plan.route_kwargs
- )(func)
-
- return self._create_standard_endpoint_decorator(
- path=plan.path,
- methods=plan.methods,
- max_upload_file_size_mb=plan.max_upload_file_size_mb,
- args=plan.route_args,
- kwargs=plan.route_kwargs
- )(func)
+ return self._create_task_endpoint_decorator(plan)(func)
+
+ return self._create_standard_endpoint_decorator(plan)(func)
return decorator
@@ -381,62 +365,6 @@ def _create_endpoint_plan(
route_args=route_args,
route_kwargs=route_kwargs,
)
-
- def _register_llm_endpoint(self, func: Callable, plan: EndpointExecutionPlan) -> Callable:
- """Register a FastAPI LLM endpoint from an endpoint execution plan."""
- req_model = plan.request_model
- res_model = plan.response_model
- endpoint_type = plan.endpoint_type
-
- if req_model is None or res_model is None or endpoint_type is None:
- raise ValueError("Invalid LLM endpoint plan: missing request/response metadata")
-
- if plan.should_use_queue:
- try:
- self._job_func_registry[func.__name__] = func
- except Exception:
- pass
-
- @functools.wraps(func)
- async def _unified_worker(*w_args, **w_kwargs):
- payload = next(
- (v for v in w_kwargs.values() if isinstance(v, (dict, req_model))),
- next((a for a in w_args if isinstance(a, (dict, req_model))), None)
- )
-
- if payload is None:
- raise ValueError(f"Request does not match expected model {req_model}")
-
- clean_kwargs = {k: v for k, v in w_kwargs.items() if not isinstance(v, req_model)}
- openai_req = self._prepare_llm_payload(req_model=req_model, payload=payload)
-
- return await self.handle_llm_request(
- func=func,
- openai_req=openai_req,
- should_use_queue=plan.should_use_queue,
- res_model=res_model,
- endpoint_type=endpoint_type,
- **clean_kwargs
- )
-
- final_handler = self._prepare_func_for_media_file_upload_with_fastapi(
- _unified_worker,
- plan.max_upload_file_size_mb
- )
-
- route_kwargs = dict(plan.route_kwargs)
- if plan.should_use_queue:
- route_kwargs["response_model_exclude_none"] = True
-
- self.api_route(
- path=plan.path,
- methods=plan.active_methods,
- response_model=JobResult if plan.should_use_queue else res_model,
- *plan.route_args,
- **route_kwargs
- )(final_handler)
-
- return final_handler
async def _execute_func(self, func, **kwargs):
"""
@@ -445,7 +373,7 @@ async def _execute_func(self, func, **kwargs):
Args:
func: Function to execute
**kwargs: Arguments to pass to function
-
+
Returns:
Result of function execution
"""
@@ -460,15 +388,15 @@ async def _execute_func(self, func, **kwargs):
async def _stream_generator(self, result):
"""
Convert a sync or async generator into an async generator for StreamingResponse.
-
+
Args:
result: Generator (sync or async) that yields streaming chunks
-
+
Yields:
Streaming chunks as strings or bytes
"""
import asyncio
-
+
if inspect.isasyncgen(result):
# Async generator - yield directly
async for chunk in result:
@@ -507,62 +435,88 @@ def _determine_queue_usage(self, use_queue: bool = None, path: str = None) -> bo
return use_queue
return self.job_queue is not None
-
- def _determine_generator_fun(self, func: Callable) -> bool:
+
+ def _is_streaming_endpoint(self, func: Callable) -> bool:
"""
- Determine whether the function is streaming data.
- Uses quick, robust checks for common streaming patterns.
+ Determine whether the function streams its output: it is a (async)
+ generator function, or its return annotation declares an iterator.
"""
- # 1. Direct generator functions
- if inspect.isgeneratorfunction(func) or inspect.isasyncgenfunction(func):
+ target = inspect.unwrap(func)
+ if inspect.isgeneratorfunction(target) or inspect.isasyncgenfunction(target):
return True
- # 2. Check return type hints
try:
- hints = get_type_hints(func)
- return_type = hints.get('return')
+ return_type = get_type_hints(target).get('return')
+ except Exception:
+ return False
- if return_type is not None:
- origin = getattr(return_type, '__origin__', return_type)
- streaming_types = (Generator, AsyncGenerator, Iterator, AsyncIterator)
+ if return_type is None:
+ return False
- if origin in streaming_types or return_type in streaming_types:
- return True
- except Exception:
- pass
+ origin = get_origin(return_type) or return_type
+ return inspect.isclass(origin) and issubclass(origin, (Iterator, AsyncIterator))
- # 3. Check source code for yield keyword
- try:
- source = inspect.getsource(func)
- if 'yield' in source:
- return True
- except Exception:
- pass
+ def _as_streaming_response(self, result: Any, binding: SchemaBinding) -> StreamingResponse | None:
+ """
+ Build a :class:`StreamingResponse` if the endpoint result is streamable:
+ - ``AudioFile`` / ``VideoFile`` stream their encoded bytes as raw chunks
+ with the proper media content type (OpenAI ``stream_format="audio"``);
+ - generators stream as-is: SSE for token-delta endpoints, raw bytes else.
+ Returns ``None`` when the result cannot be streamed.
+ """
+ if isinstance(result, (AudioFile, VideoFile)):
+ media_type = getattr(result, "content_type", None) or "application/octet-stream"
+ return StreamingResponse(iter_media_chunks(result), media_type=media_type)
+
+ if inspect.isgenerator(result) or inspect.isasyncgen(result):
+ media_type = "text/event-stream" if binding.tag in SSE_STREAM_TAGS else "application/octet-stream"
+ return StreamingResponse(
+ self._stream_generator(result),
+ media_type=media_type,
+ headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+ )
- return False
+ return None
- def _create_task_endpoint_decorator(self, path: str, methods: list[str] | None, max_upload_file_size_mb: int, queue_size: int, args, kwargs):
+ # ------------------------------------------------------------------
+ # Endpoint decorators (task / standard / streaming)
+ # ------------------------------------------------------------------
+ def _modify_result_decorator(self, func: Callable, plan: EndpointExecutionPlan) -> Callable:
+ """
+ Wraps responses of schema endpoints to their given Response format.
+ Serializes media-files to and other objects to JSON
+ """
+ @functools.wraps(func)
+ def sync_wrapper(*args, **kwargs):
+ result = func(*args, **kwargs)
+ if plan.schema_binding is not None:
+ result = wrap_schema_response(result, plan.schema_binding)
+ return JobResultFactory._serialize_result(result)
+ return sync_wrapper
+
+ def _create_task_endpoint_decorator(self, plan: EndpointExecutionPlan):
"""Create a decorator for task endpoints (background job execution)."""
# FastAPI route decorator (returning JobResult)
- task_kwargs = dict(kwargs)
+ task_kwargs = dict(plan.route_kwargs)
task_kwargs["response_model_exclude_none"] = True
fastapi_route_decorator = self.api_route(
- path=path,
- methods=["POST"] if methods is None else methods,
+ path=plan.path,
+ methods=plan.active_methods,
response_model=JobResult,
- *args,
+ *plan.route_args,
**task_kwargs
)
# Queue decorator
queue_decorator = super().job_queue_func(
- path=path,
- queue_size=queue_size,
- *args,
- **kwargs
+ path=plan.path,
+ queue_size=plan.queue_size,
+ *plan.route_args,
+ **plan.route_kwargs
)
def decorator(func: Callable) -> Callable:
+ func = self._modify_result_decorator(func, plan)
# Add job queue functionality and prepare for FastAPI file handling
queue_decorated = queue_decorator(func)
# Register the function so workers can execute it (dev mode).
@@ -570,47 +524,38 @@ def decorator(func: Callable) -> Callable:
self._job_func_registry[func.__name__] = func
except Exception:
pass
- upload_enabled = self._prepare_func_for_media_file_upload_with_fastapi(queue_decorated, max_upload_file_size_mb)
+
+ upload_enabled = self._prepare_func_for_media_file_upload_with_fastapi(queue_decorated, plan.max_upload_file_size_mb)
return fastapi_route_decorator(upload_enabled)
return decorator
- def _create_standard_endpoint_decorator(self, path: str, methods: list[str] | None, max_upload_file_size_mb: int, args, kwargs):
- """Create a decorator for standard endpoints (direct execution)."""
- # FastAPI route decorator
+ def _create_standard_endpoint_decorator(self, plan: EndpointExecutionPlan):
+ """Create a decorator for standard and schema endpoints (direct execution)."""
fastapi_route_decorator = self.api_route(
- path=path,
- methods=["POST"] if methods is None else methods,
- *args,
- **kwargs
+ path=plan.path,
+ methods=plan.active_methods,
+ response_model=plan.schema_binding.response_model if plan.is_schema_endpoint else None,
+ *plan.route_args,
+ **plan.route_kwargs
)
- def file_result_modification_decorator(func: Callable) -> Callable:
- """Wrap endpoint result and serialize it."""
- @functools.wraps(func)
- def sync_wrapper(*args, **kwargs):
- result = func(*args, **kwargs)
- return JobResultFactory._serialize_result(result)
- return sync_wrapper
-
def decorator(func: Callable) -> Callable:
- result_modified = file_result_modification_decorator(func)
- with_file_upload_signature = self._prepare_func_for_media_file_upload_with_fastapi(result_modified, max_upload_file_size_mb)
+ result_modified = self._modify_result_decorator(func, plan)
+ with_file_upload_signature = self._prepare_func_for_media_file_upload_with_fastapi(result_modified, plan.max_upload_file_size_mb)
return fastapi_route_decorator(with_file_upload_signature)
return decorator
- def _create_streaming_endpoint_decorator(self, path: str, methods: list[str] | None, max_upload_file_size_mb: int, args, kwargs):
+ def _create_streaming_endpoint_decorator(self, plan: EndpointExecutionPlan):
"""Create a decorator for streaming endpoints."""
- from fastapi.responses import StreamingResponse
-
- # Extract custom headers if provided
+ kwargs = dict(plan.route_kwargs)
custom_headers = kwargs.pop('response_headers', None)
fastapi_route_decorator = self.api_route(
- path=path,
- methods=["POST"] if methods is None else methods,
- *args,
+ path=plan.path,
+ methods=plan.active_methods,
+ *plan.route_args,
**kwargs
)
@@ -620,21 +565,13 @@ async def streaming_wrapper(*w_args, **w_kwargs):
result = await self._execute_func(func, *w_args, **w_kwargs)
generator = self._stream_generator(result)
- # Merge default and custom headers
- headers = {
- "Cache-Control": "no-cache",
- "X-Accel-Buffering": "no"
- }
+ headers = {"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}
if custom_headers:
headers.update(custom_headers)
- return StreamingResponse(
- generator,
- media_type="text/event-stream",
- headers=headers
- )
+ return StreamingResponse(generator, media_type="text/event-stream", headers=headers)
- with_file_upload_signature = self._prepare_func_for_media_file_upload_with_fastapi(streaming_wrapper, max_upload_file_size_mb)
+ with_file_upload_signature = self._prepare_func_for_media_file_upload_with_fastapi(streaming_wrapper, plan.max_upload_file_size_mb)
return fastapi_route_decorator(with_file_upload_signature)
return decorator
diff --git a/apipod/engine/backend/runpod/llm_mixin.py b/apipod/engine/backend/runpod/llm_mixin.py
deleted file mode 100644
index 225b8b9..0000000
--- a/apipod/engine/backend/runpod/llm_mixin.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from apipod.engine.llm.base_llm_mixin import _BaseLLMMixin
-
-
-class _RunPodLLMMixin(_BaseLLMMixin):
- """
- LLM logic specific to the RunPod Router
- """
-
- def handle_llm_request(self, func, openai_req, req_model, res_model, endpoint_type, w_args, w_kwargs):
- if isinstance(openai_req, dict):
- openai_req = req_model.model_validate(openai_req)
- w_kwargs["payload"] = openai_req
-
- should_stream = getattr(openai_req, "stream", False) and endpoint_type != "embedding"
-
- if should_stream:
- return self._yield_native_stream(func, w_args, w_kwargs)
-
- raw_res = self._execute_sync_or_async(func, w_args, w_kwargs)
- return self._wrap_llm_response(raw_res, res_model, endpoint_type, openai_req)
diff --git a/apipod/engine/backend/runpod/router.py b/apipod/engine/backend/runpod/router.py
index f48e806..6ed2b37 100644
--- a/apipod/engine/backend/runpod/router.py
+++ b/apipod/engine/backend/runpod/router.py
@@ -1,4 +1,5 @@
import asyncio
+import base64
import functools
import inspect
import traceback
@@ -11,20 +12,23 @@
from apipod.engine.jobs.job_result import JobResultFactory, JobResult
from apipod.engine.base_backend import _BaseBackend
from apipod.engine.files.base_file_mixin import _BaseFileHandlingMixin
-from apipod.engine.backend.runpod.llm_mixin import _RunPodLLMMixin
+from apipod.engine.backend.schema_resolve import (
+ iter_media_chunks,
+ wrap_schema_response,
+)
from apipod.engine.utils import normalize_name
from apipod.common.settings import APIPOD_PROVIDER, APIPOD_PORT, DEFAULT_DATE_TIME_FORMAT
+from media_toolkit import AudioFile, VideoFile
-class SocaityRunpodRouter(_BaseBackend, _BaseFileHandlingMixin, _RunPodLLMMixin):
+class SocaityRunpodRouter(_BaseBackend, _BaseFileHandlingMixin):
"""
Adds routing functionality for the runpod serverless framework.
Provides enhanced file handling and conversion capabilities.
"""
def __init__(self, title: str = "APIPod for ", summary: str = None, *args, **kwargs):
super().__init__(title=title, summary=summary, *args, **kwargs)
- _RunPodLLMMixin.__init__(self)
self.routes = {} # routes are organized like {"ROUTE_NAME": "ROUTE_FUNCTION"}
@@ -37,36 +41,25 @@ def endpoint(self, path: str = None, use_queue: bool = None, *args, **kwargs):
path = normalize_name(path, preserve_paths=True).strip("/")
def decorator(func: Callable) -> Callable:
- # 1. Auto-Detection
- req_model, res_model, endpoint_type = self._get_llm_config(func)
+ # binding = get_schema_binding(func)
@functools.wraps(func)
def wrapper(*w_args, **w_kwargs):
self.status = constants.SERVER_HEALTH.BUSY
-
try:
- if req_model:
- payload = w_kwargs.get("payload", None)
-
- openai_req = self._prepare_llm_payload(
- req_model=req_model,
- payload=payload
- )
-
- w_kwargs["payload"] = openai_req
-
- return self.handle_llm_request(
- func=func,
- openai_req=openai_req,
- req_model=req_model,
- res_model=res_model,
- endpoint_type=endpoint_type,
- w_args=w_args,
- w_kwargs=w_kwargs
- )
-
- # Default execution for standard endpoints
- return self._execute_sync_or_async(func, w_args, w_kwargs)
+ # if binding is None:
+ # return self._execute_sync_or_async(func, w_args, w_kwargs)
+
+ # request = prepare_schema_call(binding, w_kwargs)
+ result = self._execute_sync_or_async(func, w_args, w_kwargs)
+
+ if getattr(request, "stream", False):
+ if isinstance(result, (AudioFile, VideoFile)):
+ return self._yield_native_stream(iter_media_chunks(result))
+ if inspect.isgenerator(result) or inspect.isasyncgen(result):
+ return self._yield_native_stream(result)
+
+ return wrap_schema_response(result, binding)
finally:
self.status = constants.SERVER_HEALTH.RUNNING
@@ -74,45 +67,34 @@ def wrapper(*w_args, **w_kwargs):
return wrapper
return decorator
- def _yield_native_stream(self, func, args, kwargs):
- """Bridge for RunPod native generator streaming."""
+ @staticmethod
+ def _encode_stream_chunk(chunk) -> str:
+ """RunPod transports everything as JSON: binary chunks are base64-encoded."""
+ if isinstance(chunk, bytes):
+ return base64.b64encode(chunk).decode("ascii")
+ return chunk if isinstance(chunk, str) else str(chunk)
+
+ def _yield_native_stream(self, result):
+ """Bridge a stream result (sync/async generator or StreamingResponse) into RunPod's native stream."""
from starlette.responses import StreamingResponse
-
- # Execute the function
- result = self._execute_sync_or_async(func, args, kwargs)
-
+
# If it's a StreamingResponse (shouldn't happen in RunPod, but handle it)
if isinstance(result, StreamingResponse):
- body_iterator = result.body_iterator
-
- if inspect.isasyncgen(body_iterator):
- while True:
- try:
- chunk = self._run_in_loop(body_iterator.__anext__())
- yield chunk if isinstance(chunk, (str, bytes)) else str(chunk)
- except StopAsyncIteration:
- break
- else:
- for chunk in body_iterator:
- yield chunk if isinstance(chunk, (str, bytes)) else str(chunk)
- return
-
- # Handle async generators
+ result = result.body_iterator
+
if inspect.isasyncgen(result):
while True:
try:
- chunk = self._run_in_loop(result.__anext__())
- yield chunk if isinstance(chunk, (str, bytes)) else str(chunk)
+ yield self._encode_stream_chunk(self._run_in_loop(result.__anext__()))
except StopAsyncIteration:
break
return
-
- # Handle sync generators
- if inspect.isgenerator(result):
+
+ if inspect.isgenerator(result) or hasattr(result, "__iter__"):
for chunk in result:
- yield chunk if isinstance(chunk, (str, bytes)) else str(chunk)
+ yield self._encode_stream_chunk(chunk)
return
-
+
# Not a generator - shouldn't happen for streaming
raise TypeError(f"Expected generator for streaming, got {type(result)}")
@@ -229,18 +211,18 @@ def _router(self, path, job, **kwargs):
def _execute_route_function(self, route_function, kwargs):
"""
Execute a route function, handling both sync and async functions.
-
+
Args:
route_function: The function to execute
kwargs: Keyword arguments to pass to the function
-
+
Returns:
The result of the function execution
"""
if not inspect.iscoroutinefunction(route_function):
# Synchronous function - simple execution
return route_function(**kwargs)
-
+
# Async function - need event loop handling
try:
loop = asyncio.get_event_loop()
diff --git a/apipod/engine/backend/schema_resolve.py b/apipod/engine/backend/schema_resolve.py
new file mode 100644
index 0000000..12a6d4b
--- /dev/null
+++ b/apipod/engine/backend/schema_resolve.py
@@ -0,0 +1,235 @@
+"""
+Registry and backend-neutral helpers for standardized (OpenAI-compatible) schemas.
+
+``SCHEMA_REGISTRY`` is the single source of truth: it maps every supported
+request schema to how it is served (response model + semantic tag). Everything
+else derives from it:
+
+- :func:`resolve_request_model` / :func:`get_schema_binding` detect schema
+ endpoints by annotation (used by routers and signature policies),
+- module-level helpers prepare the incoming request (validation + nested-media
+ parsing) and wrap raw endpoint results into response models.
+
+Schemas are a stable part of APIPod, not an optional extension. Routers keep the
+normal decorator pipeline; schema helpers are called from the existing standard,
+task and streaming decorators where needed.
+
+Note: response ``id`` fields are intentionally NOT populated here — identifiers
+belong to the ``JobResult`` envelope produced by the platform, not to the model
+schemas themselves.
+"""
+
+import inspect
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from types import UnionType
+from typing import Any, Callable, Iterator, Optional, Type, Union, get_args, get_origin
+
+from media_toolkit import MediaFile
+
+from apipod.common.schemas import *
+from apipod.engine.files.base_file_mixin import parse_schema_media_fields
+
+
+@dataclass(frozen=True)
+class SchemaEndpointSpec:
+ """How a registered request schema is served."""
+ response_model: Type
+ tag: str
+
+
+# Single source of truth for all standardized schemas. Adding a schema here
+# automatically enables endpoint detection, body-parameter policies and
+# response wrapping everywhere.
+SCHEMA_REGISTRY: dict[Type, SchemaEndpointSpec] = {
+ ChatCompletionRequest: SchemaEndpointSpec(ChatCompletionResponse, "chat"),
+ CompletionRequest: SchemaEndpointSpec(CompletionResponse, "completion"),
+ EmbeddingRequest: SchemaEndpointSpec(EmbeddingResponse, "embedding"),
+ ImageGenerationRequest: SchemaEndpointSpec(ImageGenerationResponse, "image_generation"),
+ VideoGenerationRequest: SchemaEndpointSpec(VideoGenerationResponse, "video_generation"),
+ TranscriptionRequest: SchemaEndpointSpec(TranscriptionResponse, "transcription"),
+ SpeechRequest: SchemaEndpointSpec(SpeechResponse, "speech"),
+ CreateVoiceRequest: SchemaEndpointSpec(VoiceResponse, "voice"),
+ VoiceConversionRequest: SchemaEndpointSpec(VoiceConversionResponse, "voice_conversion"),
+ Generation3DRequest: SchemaEndpointSpec(Generation3DResponse, "generation_3d"),
+ VisionRequest: SchemaEndpointSpec(VisionResponse, "vision"),
+ MultimodalEmbeddingRequest: SchemaEndpointSpec(MultimodalEmbeddingResponse, "embedding_multimodal"),
+}
+
+# Tags whose streams are token deltas (served as server-sent events). Media
+# tags stream raw encoded bytes instead, matching OpenAI's stream_format="audio".
+SSE_STREAM_TAGS = {"chat", "completion", "transcription"}
+
+
+@dataclass(frozen=True)
+class SchemaBinding:
+ """Resolved binding between an endpoint function parameter and a registered schema."""
+ param_name: str
+ request_model: Type
+ response_model: Type
+ tag: str
+
+
+def _registry_spec(request_model: Type) -> Optional[SchemaEndpointSpec]:
+ """Find the registry spec for a request model, honoring subclasses of registered schemas."""
+ if not inspect.isclass(request_model):
+ return None
+ for cls in request_model.__mro__:
+ if cls in SCHEMA_REGISTRY:
+ return SCHEMA_REGISTRY[cls]
+ return None
+
+
+def resolve_request_model(annotation: Any) -> Optional[Type]:
+ """
+ Resolve an annotation to a registered request schema (or a subclass of one).
+ Supports direct annotations and Optional/Union annotations.
+ """
+ candidates = [annotation]
+ if get_origin(annotation) in (Union, UnionType):
+ candidates = [arg for arg in get_args(annotation) if arg is not type(None)]
+
+ for candidate in candidates:
+ if _registry_spec(candidate) is not None:
+ return candidate
+ return None
+
+
+def get_schema_binding(func: Callable) -> Optional[SchemaBinding]:
+ """
+ Detect the schema-typed parameter of an endpoint function by annotation — the
+ parameter may have any name, mirroring how ``JobProgress`` is detected.
+
+ Returns ``None`` for plain (non-schema) functions. When a schema parameter is
+ found, the rest of the signature is validated: a schema endpoint must take the
+ request schema as its only user input (``job_progress`` and framework
+ dependencies aside), so that the whole request lives in the schema body.
+ """
+ for param in inspect.signature(func).parameters.values():
+ request_model = resolve_request_model(param.annotation)
+ if request_model is None:
+ continue
+
+ spec = _registry_spec(request_model)
+ binding = SchemaBinding(
+ param_name=param.name,
+ request_model=request_model,
+ response_model=spec.response_model,
+ tag=spec.tag,
+ )
+ _validate_schema_endpoint_signature(func, binding)
+ return binding
+ return None
+
+
+def _is_injected_param(param: inspect.Parameter) -> bool:
+ """True for parameters the framework supplies (job progress, self/cls)."""
+ return (
+ param.name in ("self", "cls", "job_progress")
+ or "JobProgress" in str(param.annotation)
+ )
+
+
+def _validate_schema_endpoint_signature(func: Callable, binding: SchemaBinding) -> None:
+ """Reject schema endpoints that declare user parameters besides the request schema."""
+ extra = [
+ param.name
+ for param in inspect.signature(func).parameters.values()
+ if param.name != binding.param_name and not _is_injected_param(param)
+ ]
+ if extra:
+ raise TypeError(
+ f"Schema endpoint '{func.__name__}' must take the request schema "
+ f"'{binding.request_model.__name__}' as its only parameter, but also declares "
+ f"{extra}. Move these inputs into the schema (or a subclass of it)."
+ )
+
+
+def prepare_schema_call(binding: SchemaBinding, call_kwargs: dict):
+ """
+ Replace the raw request body in call kwargs with a validated schema object
+ and return that object.
+ """
+ payload = call_kwargs.get(binding.param_name)
+ if payload is None:
+ raise ValueError(
+ f"Request body for parameter '{binding.param_name}' "
+ f"({binding.request_model.__name__}) is missing"
+ )
+
+ # Prepare the request object
+ if isinstance(payload, binding.request_model):
+ request = payload
+ elif isinstance(payload, dict):
+ request = binding.request_model.model_validate(payload)
+ else:
+ raise ValueError(f"Invalid payload for {binding.request_model.__name__}: {type(payload).__name__}")
+ request = parse_schema_media_fields(request)
+
+ call_kwargs[binding.param_name] = request
+ return request
+
+
+def wrap_schema_response(result: Any, binding: SchemaBinding) -> Any:
+ """
+ Wrap a raw endpoint result into the response model of the binding.
+
+ Response models share a uniform envelope (``created``, ``model``) where
+ applicable. Raw results (strings, vectors, media files, dicts) are normalized
+ into the target response model.
+ """
+ response_model = binding.response_model
+ if isinstance(result, response_model):
+ return result
+
+ payload = _normalize_response_model(result, response_model)
+ payload = _apply_envelope(payload, response_model)
+ return response_model.model_validate(payload)
+
+
+def _normalize_response_model(result: Any, response_model: Type) -> dict:
+ """
+ Normalize raw endpoint results into a dictionary matching the response model.
+ Handles convenient raw shapes for chat, completion, embedding and transcription.
+ """
+ if response_model is ChatCompletionResponse and isinstance(result, str):
+ result = {"choices": [{"index": 0, "message": {"content": result}, "finish_reason": "stop"}]}
+ elif response_model is CompletionResponse and isinstance(result, str):
+ result = {"choices": [{"text": result, "index": 0, "finish_reason": "stop"}]}
+ elif response_model is EmbeddingResponse and isinstance(result, list):
+ vectors = result if result and isinstance(result[0], (list, tuple)) else [result]
+ result = {"data": [{"embedding": list(vector), "index": i} for i, vector in enumerate(vectors)]}
+ elif response_model is TranscriptionResponse and isinstance(result, str):
+ result = {"text": result}
+
+ return _normalize_result(result, response_model)
+
+
+def _normalize_result(result: Any, response_model: Type) -> dict:
+ """Bring raw results into dict form; media files are lifted into the `data` list."""
+ if isinstance(result, MediaFile):
+ return {"data": [result]}
+ if isinstance(result, (list, tuple)) and result and all(isinstance(item, MediaFile) for item in result):
+ return {"data": list(result)}
+ if isinstance(result, dict):
+ return dict(result)
+ raise ValueError(
+ f"Cannot wrap a result of type {type(result).__name__} into {response_model.__name__}. "
+ f"Return a {response_model.__name__}, a dict matching its fields, or a media-toolkit file."
+ )
+
+
+def _apply_envelope(payload: dict, response_model: Type) -> dict:
+ """Fill the shared response envelope (``created`` timestamp, request ``model``) when applicable."""
+ fields = response_model.model_fields
+ if "created" in fields and payload.get("created") is None:
+ payload["created"] = int(datetime.now(timezone.utc).timestamp())
+
+ return payload
+
+
+def iter_media_chunks(media_file: MediaFile, chunk_size: int = 64 * 1024) -> Iterator[bytes]:
+ """Yield the encoded bytes of a media file in chunks (for raw media streaming)."""
+ buffer = media_file.to_bytes_io()
+ while chunk := buffer.read(chunk_size):
+ yield chunk
diff --git a/apipod/engine/endpoint_config.py b/apipod/engine/endpoint_config.py
index 4f198ae..225a626 100644
--- a/apipod/engine/endpoint_config.py
+++ b/apipod/engine/endpoint_config.py
@@ -1,6 +1,8 @@
from dataclasses import dataclass
from typing import Any, Callable
+from apipod.engine.backend.schema_resolve import SchemaBinding, get_schema_binding
+
@dataclass(frozen=True)
class EndpointExecutionPlan:
@@ -18,14 +20,12 @@ class EndpointExecutionPlan:
queue_size: int
route_args: tuple[Any, ...]
route_kwargs: dict[str, Any]
- request_model: type | None = None
- response_model: type | None = None
- endpoint_type: str | None = None
+ schema_binding: SchemaBinding | None = None
is_streaming: bool = False
@property
- def is_llm(self) -> bool:
- return self.request_model is not None
+ def is_schema_endpoint(self) -> bool:
+ return self.schema_binding is not None
@property
def active_methods(self) -> list[str]:
@@ -36,8 +36,8 @@ class FastApiEndpointConfigurator:
"""
Builds endpoint execution plans for the FastAPI backend.
- This component configures endpoint behavior (LLM, streaming, queue)
- independent from provider mechanics (FastAPI routing).
+ This component configures endpoint behavior (schema dispatch, streaming,
+ queue) independent from provider mechanics (FastAPI routing).
"""
def __init__(self, router):
@@ -55,8 +55,8 @@ def build_plan(
route_args: tuple[Any, ...],
route_kwargs: dict[str, Any],
) -> EndpointExecutionPlan:
- request_model, response_model, endpoint_type = self._router._get_llm_config(func)
- is_streaming = bool(request_model is None and self._router._determine_generator_fun(func))
+ schema_binding = get_schema_binding(func)
+ is_streaming = bool(schema_binding is None and self._router._is_streaming_endpoint(func))
return EndpointExecutionPlan(
path=path,
@@ -66,8 +66,6 @@ def build_plan(
queue_size=queue_size,
route_args=route_args,
route_kwargs=route_kwargs,
- request_model=request_model,
- response_model=response_model,
- endpoint_type=endpoint_type,
+ schema_binding=schema_binding,
is_streaming=is_streaming,
)
diff --git a/apipod/engine/files/base_file_mixin.py b/apipod/engine/files/base_file_mixin.py
index eeaf24f..3e0cae0 100644
--- a/apipod/engine/files/base_file_mixin.py
+++ b/apipod/engine/files/base_file_mixin.py
@@ -1,14 +1,67 @@
import functools
import inspect
from types import UnionType
-from typing import Any, Union, get_args, get_origin, Callable, List, Type
+from typing import Any, Optional, Union, get_args, get_origin, Callable, List, Type
-from media_toolkit import media_from_any, MediaFile, MediaList, MediaDict
+from pydantic import BaseModel
+
+from media_toolkit import media_from_any, MediaFile, MediaList, MediaDict, ImageFile, AudioFile, VideoFile
from apipod.engine.signatures.upload import is_param_media_toolkit_file
-from apipod.engine.jobs.job_result import FileModel
+from apipod.common.schemas.media_files import FileModel, ImageFileModel, AudioFileModel, VideoFileModel
from apipod.common.exceptions import FileUploadException
+# Wire-format pydantic file models mapped to the media-toolkit type they parse into.
+_FILE_MODEL_MEDIA_TYPES: dict = {
+ ImageFileModel: ImageFile,
+ AudioFileModel: AudioFile,
+ VideoFileModel: VideoFile,
+ FileModel: MediaFile,
+}
+
+
+def _media_type_for_file_model(file_model_cls: Type) -> Type:
+ """Resolve the media-toolkit type a FileModel subclass should be parsed into."""
+ for cls in file_model_cls.__mro__:
+ if cls in _FILE_MODEL_MEDIA_TYPES:
+ return _FILE_MODEL_MEDIA_TYPES[cls]
+ return MediaFile
+
+
+def _parse_file_model_value(value: Any) -> Any:
+ """Convert FileModel instances (also inside lists) into parsed media-toolkit objects."""
+ if isinstance(value, FileModel):
+ return media_from_any(
+ data=value.model_dump(include={"file_name", "content_type", "content"}),
+ type_hint=_media_type_for_file_model(type(value)),
+ use_temp_file=True,
+ temp_dir=None,
+ allow_reads_from_disk=False,
+ )
+ if isinstance(value, list):
+ return [_parse_file_model_value(item) for item in value]
+ return value
+
+
+def parse_schema_media_fields(schema_obj):
+ """
+ Replace FileModel-typed fields of a validated request schema with parsed
+ media-toolkit objects, so endpoint functions receive ready-to-use media —
+ exactly as method-level ``def endpoint(image: ImageFile)`` parameters do.
+
+ Lives in the file-handling layer because it is the schema-side counterpart
+ of the parameter-level upload conversion (``_convert_param_to_media_file``).
+ """
+ for field_name in type(schema_obj).model_fields:
+ value = getattr(schema_obj, field_name, None)
+ parsed = _parse_file_model_value(value)
+ if parsed is not value:
+ # The runtime type (e.g. ImageFile) intentionally differs from the wire
+ # annotation (ImageFileModel); bypass pydantic's assignment validation.
+ object.__setattr__(schema_obj, field_name, parsed)
+ return schema_obj
+
+
class _BaseFileHandlingMixin:
"""
Base mixin for handling file uploads and parameter conversions across different deployment environments.
@@ -230,6 +283,49 @@ def _read_upload_files(self, files: dict, media_types: dict, *args, **kwargs) ->
raise ValueError(f"Could not parse file {key}. Check if the file is correct. Error: {str(e)}")
return converted_files
+ def _resolve_schema_annotation(self, annotation: Any) -> Optional[Type]:
+ """
+ Resolve an annotation (also ``Optional``/``Union``) to a pydantic request
+ schema whose nested FileModel fields should be parsed.
+
+ FileModel variants are themselves media handled by the parameter-level upload
+ path (``_convert_param_to_media_file``), so they are excluded here.
+ """
+ candidates = [annotation]
+ if get_origin(annotation) in (Union, UnionType):
+ candidates = [arg for arg in get_args(annotation) if arg is not type(None)]
+
+ for candidate in candidates:
+ if inspect.isclass(candidate) and issubclass(candidate, BaseModel) and not issubclass(candidate, FileModel):
+ return candidate
+ return None
+
+ def _parse_schema_params(self, schema_params: set, named_args: dict) -> dict:
+ """
+ Parse nested FileModel fields inside request-schema arguments.
+
+ Called by ``_handle_file_uploads`` before the media-param conversion so that
+ endpoint functions always receive ready-to-use media-toolkit objects regardless
+ of whether files arrive as top-level parameters or nested inside a schema.
+
+ Args:
+ schema_params: Parameter names whose annotation resolved to a pydantic schema.
+ named_args: Current mapping of parameter name → value for this call.
+
+ Returns:
+ Mapping of parameter name → schema instance with media fields replaced.
+ """
+ parsed = {}
+ for name in schema_params:
+ value = named_args.get(name)
+ if value is None:
+ continue
+ try:
+ parsed[name] = parse_schema_media_fields(value)
+ except Exception as e:
+ raise FileUploadException(message=f"Could not parse media fields of '{name}': {e}")
+ return parsed
+
def _handle_file_uploads(self, func: Callable) -> Callable:
"""
Wrap a function to handle file uploads and conversions.
@@ -243,7 +339,14 @@ def _handle_file_uploads(self, func: Callable) -> Callable:
"""
sig = inspect.signature(func)
media_params = self._get_media_params(sig)
-
+
+ # Parameters typed as a pydantic request schema whose nested FileModel fields
+ # must be parsed into media-toolkit objects before the endpoint executes.
+ schema_params = {
+ name for name, annot in self._sig_to_annotations(sig).items()
+ if name not in media_params and self._resolve_schema_annotation(annot) is not None
+ }
+
param_names = list(sig.parameters.keys())
# Check if the original function is async
@@ -254,6 +357,9 @@ async def file_upload_wrapper(*args, **kwargs):
named_args = {param_names[i]: arg for i, arg in enumerate(args) if i < len(param_names)}
named_args.update(kwargs)
+ # Parse nested media inside schema parameters
+ named_args.update(self._parse_schema_params(schema_params, named_args))
+
# Convert media-related parameters
files_to_process = {
param_name: param_value
@@ -268,7 +374,7 @@ async def file_upload_wrapper(*args, **kwargs):
# Update arguments with converted files
named_args.update(processed_files)
-
+
# AWAIT the async function
return await func(**named_args)
else:
@@ -278,6 +384,9 @@ def file_upload_wrapper(*args, **kwargs):
named_args = {param_names[i]: arg for i, arg in enumerate(args) if i < len(param_names)}
named_args.update(kwargs)
+ # Parse nested media inside schema parameters
+ named_args.update(self._parse_schema_params(schema_params, named_args))
+
# Convert media-related parameters
files_to_process = {
param_name: param_value
@@ -292,7 +401,7 @@ def file_upload_wrapper(*args, **kwargs):
# Update arguments with converted files
named_args.update(processed_files)
-
+
# Call the sync function directly
return func(**named_args)
diff --git a/apipod/engine/jobs/job_result.py b/apipod/engine/jobs/job_result.py
index 11c0288..b380970 100644
--- a/apipod/engine/jobs/job_result.py
+++ b/apipod/engine/jobs/job_result.py
@@ -2,66 +2,14 @@
from io import BytesIO
from typing import Any, List, Optional, Union
-from pydantic import BaseModel, AnyUrl
+from pydantic import BaseModel
from apipod.common.settings import DEFAULT_DATE_TIME_FORMAT
from apipod.engine.jobs.base_job import JOB_STATUS, BaseJob
from apipod.engine.signatures.upload import is_param_media_toolkit_file
from media_toolkit import IMediaContainer
from media_toolkit.utils.data_type_utils import is_file_model_dict
-
-
-class FileModel(BaseModel):
- file_name: str
- content_type: str
- content: Union[str, AnyUrl] # base64 encoded or url
- max_size_mb: Optional[float] = 4000
-
- class Config:
- json_schema_extra = {
- "x-media-type": "MediaFile",
- "example": {
- "file_name": "example.csv",
- "content_type": "text/csv",
- "content": "https://example.com/example.csv",
- }
- }
-
-
-class ImageFileModel(FileModel):
- class Config:
- json_schema_extra = {
- "x-media-type": "ImageFile",
- "example": {
- "file_name": "example.png",
- "content_type": "image/png",
- "content": "base64 encoded image data",
- }
- }
-
-
-class AudioFileModel(FileModel):
- class Config:
- json_schema_extra = {
- "x-media-type": "AudioFile",
- "example": {
- "file_name": "example.mp3",
- "content_type": "audio/mpeg",
- "content": "base64 encoded audio data",
- }
- }
-
-
-class VideoFileModel(FileModel):
- class Config:
- json_schema_extra = {
- "x-media-type": "VideoFile",
- "example": {
- "file_name": "example.mp4",
- "content_type": "video/mp4",
- "content": "base64 encoded video data",
- }
- }
+from apipod.common.schemas.media_files import FileModel
def _job_status_to_public(status: Any) -> Optional[str]:
@@ -208,6 +156,11 @@ def _serialize_result(data: Any) -> Union[FileModel, List[FileModel], list, str,
except Exception:
pass
+ # Pydantic schema responses (e.g. SpeechResponse): dump to a dict and
+ # recurse so nested media files / FileModels serialize correctly too.
+ if isinstance(data, BaseModel):
+ return JobResultFactory._serialize_result(data.model_dump())
+
if isinstance(data, list):
return [JobResultFactory._serialize_result(item) for item in data]
diff --git a/apipod/engine/schema_extension/__init__.py b/apipod/engine/schema_extension/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/apipod/engine/schema_extension/schema_mixin.py b/apipod/engine/schema_extension/schema_mixin.py
deleted file mode 100644
index ae3c1f3..0000000
--- a/apipod/engine/schema_extension/schema_mixin.py
+++ /dev/null
@@ -1,166 +0,0 @@
-from types import UnionType
-from typing import Callable, Type, Any, get_args, get_origin, Union
-import uuid
-from datetime import datetime, timezone
-import inspect
-from apipod.common.schemas import (
- ChatCompletionRequest, ChatCompletionResponse, ChatCompletionChoice,
- CompletionRequest, CompletionResponse, CompletionChoice,
- EmbeddingRequest, EmbeddingResponse, EmbeddingData,
- ImageGenerationRequest, ImageGenerationResponse, ImageGenerationData,
- VideoGenerationRequest, VideoGenerationResponse, VideoGenerationData,
- AudioRequest, AudioResponse, AudioData,
- Generation3DRequest, Generation3DResponse, Generation3DData,
- VisionRequest, VisionResponse, VisionData,
- MultimodalEmbeddingRequest, MultimodalEmbeddingResponse, MultimodalEmbeddingData,
-)
-
-
-class _SchemaExtMixin:
- """
- Mixin class for Base LLM functionality
- """
-
- def __init__(self, *args, **kwargs):
- self._llm_configs = {
- ChatCompletionRequest: (ChatCompletionResponse, "chat"),
- CompletionRequest: (CompletionResponse, "completion"),
- EmbeddingRequest: (EmbeddingResponse, "embedding"),
- ImageGenerationRequest: (ImageGenerationResponse, "image_generation"),
- VideoGenerationRequest: (VideoGenerationResponse, "video_generation"),
- AudioRequest: (AudioResponse, "audio"),
- Generation3DRequest: (Generation3DResponse, "generation_3d"),
- VisionRequest: (VisionResponse, "vision"),
- MultimodalEmbeddingRequest: (MultimodalEmbeddingResponse, "embedding_multimodal"),
- }
- self._supported_llm_request_models = tuple(self._llm_configs.keys())
-
- def _resolve_supported_llm_request_model(self, annotation: Any):
- """Resolve direct/optional request model annotations to a supported LLM request model."""
- if annotation in self._llm_configs:
- return annotation
-
- origin = get_origin(annotation)
- if origin in (Union, UnionType):
- for arg in get_args(annotation):
- if arg is type(None):
- continue
- if arg in self._llm_configs:
- return arg
-
- return None
-
- def _get_llm_config(self, func: Callable):
- sig = inspect.signature(func)
- for param in sig.parameters.values():
- req_model = self._resolve_supported_llm_request_model(param.annotation)
- if req_model is not None:
- res_model, endpoint_type = self._llm_configs[req_model]
- return req_model, res_model, endpoint_type
- return None, None, None
-
- def _prepare_llm_payload(self, req_model: Type, payload: Any) -> Any:
- """
- Prepare the LLM request payload.
- """
- if isinstance(payload, req_model):
- return payload
- elif isinstance(payload, dict):
- return req_model.model_validate(payload)
- else:
- raise ValueError(f"Invalid payload type for {req_model}: {type(payload)}")
-
- def _wrap_llm_response(self, result: Any, response_model: Type, endpoint_type: str, openai_req: Any) -> Any:
- """
- Wrap the raw result into the appropriate LLM response model.
- """
- if isinstance(result, response_model):
- return result
-
- model_name = getattr(openai_req, "model", "unknown-model")
- ts, uid = int(datetime.now(timezone.utc).timestamp()), uuid.uuid4().hex[:8]
-
- if endpoint_type == "chat":
- return response_model(
- id=f"chatcmpl-{ts}-{uid}",
- object="chat.completion",
- created=ts,
- model=model_name,
- choices=[ChatCompletionChoice(**choice) for choice in result["choices"]],
- usage=result.get("usage")
- )
- elif endpoint_type == "completion":
- return response_model(
- id=f"cmpl-{ts}-{uid}",
- object="text.completion",
- created=ts,
- model=model_name,
- choices=[CompletionChoice(**choice) for choice in result["choices"]],
- usage=result.get("usage")
- )
- elif endpoint_type == "embedding":
- # FRICTION #15 fix: the outer object on EmbeddingResponse is "list"
- # (each item in data carries object="embedding"). The schema declares
- # object: Literal["list"]; passing "embedding" here used to fail
- # Pydantic validation, which is why qwen-models constructs the
- # response by hand. After this fix, the auto-wrap path is safe.
- return response_model(
- object="list",
- data=[EmbeddingData(**data) for data in result["data"]],
- model=model_name,
- usage=result.get("usage")
- )
- elif endpoint_type == "image_generation":
- return response_model(
- id=f"imggen-{ts}-{uid}",
- object="image_generation",
- created=ts,
- model=model_name,
- data=[ImageGenerationData(**item) for item in result["data"]],
- usage=result.get("usage")
- )
- elif endpoint_type == "video_generation":
- return response_model(
- id=f"vidgen-{ts}-{uid}",
- object="video_generation",
- created=ts,
- model=model_name,
- data=[VideoGenerationData(**item) for item in result["data"]],
- usage=result.get("usage")
- )
- elif endpoint_type == "audio":
- return response_model(
- id=f"aud-{ts}-{uid}",
- object="audio",
- created=ts,
- model=model_name,
- data=[AudioData(**item) for item in result["data"]],
- usage=result.get("usage")
- )
- elif endpoint_type == "generation_3d":
- return response_model(
- id=f"gen3d-{ts}-{uid}",
- object="generation_3d",
- created=ts,
- model=model_name,
- data=[Generation3DData(**item) for item in result["data"]],
- usage=result.get("usage")
- )
- elif endpoint_type == "vision":
- return response_model(
- id=f"vis-{ts}-{uid}",
- object="vision",
- created=ts,
- model=model_name,
- data=[VisionData(**item) for item in result["data"]],
- usage=result.get("usage")
- )
- elif endpoint_type == "embedding_multimodal":
- return response_model(
- object="list",
- data=[MultimodalEmbeddingData(**item) for item in result["data"]],
- model=model_name,
- usage=result.get("usage")
- )
- else:
- raise ValueError(f"Unknown endpoint type: {endpoint_type}")
\ No newline at end of file
diff --git a/apipod/engine/signatures/policies.py b/apipod/engine/signatures/policies.py
index ce4c6d9..cc255df 100644
--- a/apipod/engine/signatures/policies.py
+++ b/apipod/engine/signatures/policies.py
@@ -1,10 +1,14 @@
-from types import UnionType
-from typing import Any, Union, get_args, get_origin
-
import inspect
+from typing import Any
+
from fastapi import Body, Form
-from apipod.common.schemas import SUPPORTED_LLM_REQUEST_SCHEMAS
+from apipod.engine.backend.schema_resolve import SCHEMA_REGISTRY, resolve_request_model
+
+# Request schemas that are interpreted as JSON bodies by router decorators,
+# even when endpoint authors do not specify Body(...). Derived from the
+# schema registry so the two can never drift apart.
+SUPPORTED_REQUEST_SCHEMAS = tuple(SCHEMA_REGISTRY)
class FastAPISignaturePolicies:
@@ -26,33 +30,22 @@ def is_fastapi_dependency(parameter: inspect.Parameter) -> bool:
return module.startswith("fastapi") or module.startswith("starlette")
@staticmethod
- def is_supported_llm_request_schema(annotation: Any) -> bool:
+ def is_supported_request_schema(annotation: Any) -> bool:
"""
- True if annotation is a supported APIPod LLM request schema.
- Supports direct and optional union annotations.
+ True if annotation is (or contains, for Optional/Union annotations) a
+ registered APIPod request schema or a subclass of one.
"""
- if annotation in SUPPORTED_LLM_REQUEST_SCHEMAS:
- return True
-
- origin = get_origin(annotation)
- if origin in (Union, UnionType):
- return any(
- arg in SUPPORTED_LLM_REQUEST_SCHEMAS
- for arg in get_args(annotation)
- if arg is not type(None)
- )
-
- return False
+ return resolve_request_model(annotation) is not None
@classmethod
def build_non_file_default(cls, annotation: Any, default: Any, is_optional: bool):
"""
Select FastAPI parameter defaults for non-file fields.
- Supported LLM request schemas are mapped to JSON body semantics. All other
+ Registered request schemas are mapped to JSON body semantics. All other
non-file parameters keep form semantics for backward compatibility.
"""
normalized_default = None if is_optional else default
- if cls.is_supported_llm_request_schema(annotation):
+ if cls.is_supported_request_schema(annotation):
return Body(default=normalized_default)
return Form(default=normalized_default)
diff --git a/docs/README_TECH.md b/docs/README_TECH.md
index 7b9c79c..fa7b482 100644
--- a/docs/README_TECH.md
+++ b/docs/README_TECH.md
@@ -1,20 +1,124 @@
+# APIPod — Technical Guide
-# Queue mixin, Job Queue and Job lifecycle
+This document explains how APIPod works internally: its architecture, the request lifecycle, and the design principles behind it. It is aimed at developers maintaining or extending the package. For usage-oriented documentation see the [main README](../README.md).
-The fastapi router backend uses the _queue_mixin to manage jobs.
-Every call to an task_endpoint will submit the job via the mixin to the job_queue.
+## What APIPod is
+APIPod is the standardized way to build AI services for the [socaity.ai](https://www.socaity.ai) catalog. It wraps FastAPI with the batteries an AI service needs — media file handling, job queues with progress reporting, serverless routing, standardized request/response schemas — while keeping the developer experience of plain FastAPI: you write a function, annotate its parameters, and decorate it with `@app.endpoint`.
-Jobs are managed by the job_queue.
-When a job is created in, it goes through a series of stages before it is completed.
+It sits in an ecosystem of three packages:
+
+| Package | Role |
+| --- | --- |
+| **APIPod** (this repo) | Server side: define and deploy AI service endpoints |
+| [media-toolkit](https://github.com/SocAIty/media-toolkit) | Shared media types (`ImageFile`, `AudioFile`, `VideoFile`, …) used for I/O on both ends |
+| [fastSDK](https://github.com/SocAIty/fastSDK) | Client side: generated SDKs that handle uploads, polling and streaming |
+
+## Package layout
+
+```
+apipod/
+├── api.py # APIPod() factory: resolves config → backend instance
+├── common/
+│ ├── settings.py # Env-driven config (APIPOD_ORCHESTRATOR / _COMPUTE / _PROVIDER, host, port)
+│ ├── constants.py # Enums: ORCHESTRATOR, COMPUTE, PROVIDER, SERVER_HEALTH
+│ └── schemas/
+│ ├── schemas.py # Standardized request/response models (OpenAI-compatible shapes)
+│ └── media_files.py # FileModel + typed variants: pydantic mirrors of media-toolkit files
+├── engine/
+│ ├── base_backend.py # Shared backend base (title, version, health)
+│ ├── endpoint_config.py # EndpointExecutionPlan + configurator (how to register an endpoint)
+│ ├── backend/
+│ │ ├── fastapi/ # SocaityFastAPIRouter + file handling, exceptions
+│ │ └── runpod/ # SocaityRunpodRouter (serverless, path-based)
+│ ├── files/ # _BaseFileHandlingMixin + parse_schema_media_fields (inputs → MediaFile)
+│ ├── jobs/ # BaseJob, JobProgress, JobResult (+ factory)
+│ ├── queue/ # JobQueue, JobStore, _QueueMixin (enqueue instead of block)
+│ └── signatures/ # Signature inspection policies (media params, Body vs Form, …)
+└── deploy/ # `apipod build`: Dockerfile generation, dependency/CUDA detection
+```
+
+## Core principles
+
+1. **One decorator, many execution modes.** Developers only write `@app.endpoint(...)`. The router inspects the function and decides how to run it. They never choose a "mode" explicitly — the signature is the contract.
+2. **Deployment is configuration, not code.** The same service file runs as a plain FastAPI app, a queued FastAPI app, or a RunPod serverless worker. `APIPod()` is a factory that picks the backend from `orchestrator` / `compute` / `provider` (constructor args or `APIPOD_*` env vars).
+3. **Media files are objects, not bytes.** Endpoint authors annotate parameters with media-toolkit types (`ImageFile`, `AudioFile`, …) and receive parsed, ready-to-use objects — regardless of whether the client sent a multipart upload, a URL, or base64.
+4. **Standardized, OpenAI-compatible schemas.** Common AI payloads (chat, completion, embeddings, image/video/audio/3D generation, vision) have canonical pydantic schemas whose wire format mirrors the OpenAI API. The shape is provider-agnostic: any model can serve them; clients written against OpenAI-compatible tooling work without translation.
+5. **Long-running work returns a job, not a blocked connection.** With a queue configured, endpoints return a `JobResult` immediately; clients poll `/status/{job_id}` (fastSDK does this automatically) and can receive progress updates via `JobProgress`.
+
+## Backend resolution
+
+`APIPod()` in `api.py` is not a class — it is a factory. It resolves the `(orchestrator, compute, provider)` triple (explicit args → env vars → defaults) against a support matrix and returns one of:
+
+- **`SocaityFastAPIRouter`** — an `APIRouter` subclass bound to a `FastAPI` app. Used for dedicated compute and for local serverless emulation (then paired with an in-memory `JobQueue` plus a background worker thread started via the app lifespan).
+- **`SocaityRunpodRouter`** — a path-based dispatcher for RunPod serverless. There is no HTTP layer: RunPod delivers a JSON job whose `input.path` selects the registered function; the router converts files, injects `JobProgress`, executes, and returns a serialized `JobResult` (or a generator for streaming). It can also synthesize an OpenAPI schema by replaying the FastAPI signature conversion, so fastSDK clients can be generated against serverless deployments too.
+
+The full configuration matrix is documented in the main README and in the `APIPod()` docstring.
+
+## The endpoint pipeline (FastAPI backend)
+
+When a function is decorated with `@app.endpoint(path)`, the router asks the `FastApiEndpointConfigurator` to build an immutable `EndpointExecutionPlan`. The plan records whether the signature contains a registered request schema, whether the function itself streams, and whether queueing is enabled.
+
+The function then goes through the normal APIPod decorator pipeline:
+
+1. **Schema binding detection** — a parameter (any name) annotated with a standardized request schema (e.g. `ChatCompletionRequest`) or a subclass becomes `plan.schema_binding`. This does not register a separate route. It only tells the downstream decorators to parse that JSON body into the schema object and wrap the final result into the registered response model.
+2. **Streaming endpoint decorator** — for generator functions, output is bridged into a `StreamingResponse` with SSE-friendly headers.
+3. **Task endpoint decorator** — when a job queue is configured (and `use_queue` is not `False`), the call enqueues a job and immediately returns a `JobResult`. Schema requests with `stream=true` are the one request-level exception: the task decorator executes them inline and streams the result instead of queueing.
+4. **Standard endpoint decorator** — direct execution. It also handles non-queued schema endpoints: parse the request schema, execute the user function, wrap the result into the schema response model, and serialize non-schema media results through `JobResultFactory`.
+
+In all cases the function then passes through the **file-handling preparation** (next section) before being handed to FastAPI's `api_route`.
+
+### File handling: two layers
+
+File support is split into a *signature* layer and a *runtime* layer.
+
+- **Signature rewriting** (`engine/backend/fastapi/file_handling_mixin.py`): media-toolkit annotations in the user's signature are rewritten so FastAPI/OpenAPI understand them. `image: ImageFile` becomes `Union[LimitedUploadFile, ImageFileModel, str]` — the client may send a multipart upload, a `FileModel` JSON object (`{file_name, content_type, content}` where content is base64 or a URL), or a plain URL/base64 string. `MediaList[...]` maps to list variants. Upload size limits are enforced via a dynamically subclassed `LimitedUploadFile`. `JobProgress` parameters are stripped from the public signature (and a dummy is injected when no queue runs).
+- **Runtime conversion** (`engine/files/base_file_mixin.py`): before the user function executes, every media-annotated argument is converted to the annotated media-toolkit type via `media_from_any` — whatever the client actually sent. The function body always receives real `MediaFile` objects. This layer is backend-agnostic and reused by the RunPod router.
+
+On the way out, `JobResultFactory._serialize_result` converts returned `MediaFile`/`MediaList`/pydantic objects back into JSON-safe `FileModel` payloads.
+
+### Standardized schemas
+
+`common/schemas/schemas.py` defines request/response pairs for: chat completion, text completion, embeddings, image generation, video generation, transcription, speech (TTS), voice creation, voice conversion, 3D generation, vision, and multimodal embeddings. `model` is optional on every request, since an APIPod service typically serves exactly one model. The audio API is split per use case, mirroring OpenAI: `TranscriptionRequest` (STT), `SpeechRequest` (TTS, with `voice` as a named voice or a cloning reference file), `CreateVoiceRequest` (voice cloning → embedding) and `VoiceConversionRequest` (voice2voice).
+
+`SCHEMA_REGISTRY` in `engine/schema_extension/schema_mixin.py` is the single source of truth: it maps each request schema to a `SchemaEndpointSpec` (response model, tag). Everything else derives from it — `engine/signatures/policies.py` builds `SUPPORTED_REQUEST_SCHEMAS` from the registry keys (schema-annotated parameters are read from the JSON body while plain parameters stay form-encoded), and both routers detect schema endpoints with `get_schema_binding`, which finds the schema-typed parameter by annotation regardless of its name (subclasses of registered schemas are also detected, so services can extend a schema with custom fields). Schema endpoints may not declare additional user parameters; put extra inputs on the schema or a schema subclass.
+
+**Nested media files**: request schemas may declare `FileModel`-typed fields (`ImageGenerationRequest.image`, `TranscriptionRequest.audio`, …). Pydantic accepts uploads, FileModel JSON objects, URLs and plain base64 strings for those fields; before the endpoint function runs, `parse_schema_media_fields` (in `engine/files/base_file_mixin.py`) replaces them with parsed media-toolkit objects — the endpoint receives a ready-to-use `ImageFile`/`AudioFile`, exactly like method-level `def endpoint(image: ImageFile)` parameters.
+
+**Response wrapping** (`wrap_schema_response`): if the function already returns the response model, it passes through. Schema helpers also accept convenient raw results (a plain string for chat/completion/transcription, raw vectors for embeddings). Everything else shares one generic path: a uniform envelope (`created`, `model`) merged with the result dict and validated by pydantic — a returned media-toolkit file is lifted into the `data` list automatically. Response IDs are not generated by APIPod schemas; IDs belong to the platform `JobResult`/socaity.ai layer.
+
+### Jobs, queue and lifecycle
+
+The local `JobQueue` (in-memory, threaded) drives the serverless emulation and dedicated-with-queue modes. A job passes through these stages:
+
+- `validate_job_before_add` — parameter/permission validation
+- `add_job` — persisted in the `JobStore`, returns immediately
+- `create_job` / `process_job` — the worker picks it up and runs the user function
+- `complete_job` — result stored, status set
+- `remove_job` — cleaned up once collected (or orphaned)
+
+`JobProgress` is the in-band progress channel: if the user function declares a `job_progress` parameter, the backend injects an implementation (`JobProgress` locally, `JobProgressRunpod` on RunPod) and `set_status(progress, message)` updates surface in `/status` polls. `JobResult` is the unified public envelope: `job_id`, `status` (`pending/processing/completed/failed`), `result`, `progress`, `message`, timing `metrics`, and hypermedia `links` (status/cancel/stream).
+
+Standard routes registered automatically: `GET /status/{job_id}`, `POST /cancel/{job_id}`, `GET /health`, and — when a stream store is configured — `GET /stream/{job_id}` (SSE).
+
+### Streaming
+
+Three streaming paths exist today:
+
+1. **Generator endpoints** — any endpoint function that is a (async) generator, or whose return annotation declares an iterator, is served as an SSE `StreamingResponse` (FastAPI) or as a native generator handed back to RunPod (`return_aggregate_stream`). Detection is annotation/inspect-based — no source-code heuristics.
+2. **Schema endpoints with `stream=true`** — the request schema carries the `stream` flag (chat, completion, transcription, speech, video generation). Streaming bypasses the queue; what gets streamed depends on what the function returns:
+ - a **generator**: streamed as-is — SSE for token-delta endpoints (chat/completion/transcription, e.g. `ChatCompletionChunk` events), raw bytes otherwise;
+ - an **`AudioFile`/`VideoFile`**: its encoded bytes are chunked into a `StreamingResponse` with the file's media content type — raw audio chunks, not SSE, matching OpenAI's `stream_format="audio"` behavior;
+ - anything else: the regular wrapped JSON response (the endpoint cannot stream).
+ On RunPod the same logic applies, but chunks pass through `_yield_native_stream`, which base64-encodes binary chunks because RunPod transports JSON.
+3. **Job streaming** — `GET /stream/{job_id}` replays chunks from a pluggable stream store while a queued job is in `streaming` state (gateway integration).
+
+### Deployment
+
+`apipod build` (see `deploy/`) scans the project (entrypoint, dependencies, CUDA requirements) and generates a Dockerfile from compatible templates. The resulting container runs unchanged on dedicated hosts, on socaity.ai, or on RunPod serverless — only the `APIPOD_*` env vars differ.
+
+## Request lifecycle, end to end
+
+A client calls `POST /tts` with a JSON body. FastAPI validates it against the rewritten signature and calls the outermost wrapper. The file-handling layer converts any media inputs into media-toolkit objects. If the endpoint is queued, the queue mixin stores the job and returns `{job_id, status: "pending", links}` — the worker thread later executes the real function, feeding `JobProgress` updates into the store. The client (typically fastSDK) polls `/status/{job_id}` until `completed` and receives the result, with any returned `AudioFile` serialized as a `FileModel`. Without a queue the same conversion happens inline and the response returns directly. On RunPod, the identical user function is reached through `handler → _router(path) → file handling → execute`, proving the core principle: the function is written once, the backend decides how it runs.
-The stages are:
-- validate_job_before_add: Used to validate parameters, user permissions, etc.
-- add_job: job gets added to job store.
-- create_job: literally creates the job. Can be overridden to use a specialized job class.
-- process_job: The main job processing function. This is where the actual work is done.
-- complete_job: Called when the job is completed. This is where the job is marked as completed and the result is stored.
-- remove_job:
- - Called when the job is removed from the JobStore and RAM.
- - This happens when the user gets the result of the job or when the job was never collected (orphant job).
diff --git a/examples/test_service_six_schemas.py b/examples/test_service_six_schemas.py
index 9d0d36d..6cd9b01 100644
--- a/examples/test_service_six_schemas.py
+++ b/examples/test_service_six_schemas.py
@@ -1,14 +1,14 @@
"""
-Minimal APIPod service used to verify media-file integration end to end for
-PR #15 (the 6 new Pydantic schemas).
+Minimal APIPod service used to verify media-file integration of the
+standardized schemas end to end.
It exposes two endpoints:
-- POST /vision uses VisionRequest, which has a REQUIRED image field
-- POST /audio uses AudioRequest, which has an OPTIONAL audio field
+- POST /vision uses VisionRequest, which has a REQUIRED image field
+- POST /transcribe uses TranscriptionRequest, which has a REQUIRED audio field
Both endpoints just inspect the incoming media file and echo back a few
-properties (size, content type, etc.) so we can confirm the file_handling
-mixin actually deserialized URL/base64 input into a real ImageFile/AudioFile.
+properties (size, content type, etc.) so we can confirm the file handling
+actually deserialized upload/URL/base64 input into a real ImageFile/AudioFile.
Run with:
python examples/test_service_six_schemas.py
@@ -16,10 +16,7 @@
"""
from apipod import APIPod
-from apipod.common.schemas import (
- VisionRequest,
- AudioRequest,
-)
+from apipod.common.schemas import TranscriptionRequest, VisionRequest
app = APIPod()
@@ -38,10 +35,10 @@ def _media_summary(media) -> dict:
@app.endpoint("/vision")
-def vision(payload: VisionRequest):
- """Echoes a label built from payload.image. The point is that the image
+def vision(request: VisionRequest):
+ """Echoes a label built from request.image. The point is that the image
arrived as an ImageFile, deserialized from upload / base64 / URL."""
- summary = _media_summary(payload.image)
+ summary = _media_summary(request.image)
return {
"data": [
{
@@ -52,17 +49,12 @@ def vision(payload: VisionRequest):
}
-@app.endpoint("/audio")
-def audio(payload: AudioRequest):
- """Echoes the audio properties (or text if no audio was sent)."""
- if payload.audio is not None:
- text = f"echo:{_media_summary(payload.audio)}"
- else:
- text = f"no-audio:text={payload.text!r}"
+@app.endpoint("/transcribe")
+def transcribe(request: TranscriptionRequest):
+ """Echoes the audio properties as the 'transcript'."""
return {
- "data": [
- {"text": text, "language": payload.language, "duration_s": None}
- ]
+ "text": f"echo:{_media_summary(request.audio)}",
+ "language": request.language,
}
diff --git a/test/llm/test_llm_mock.py b/test/llm/test_llm_mock.py
index 2695feb..fb09cfc 100644
--- a/test/llm/test_llm_mock.py
+++ b/test/llm/test_llm_mock.py
@@ -6,7 +6,6 @@
from typing import List, Union
from datetime import datetime
import time
-import json
import uuid
import random
@@ -14,13 +13,10 @@
from apipod.common import schemas
from fastapi import FastAPI
-# ============================================================================
-# Mock Model Wrapper
-# ============================================================================
class MockModel:
"""A mock model that returns random strings to mimic an LLM."""
-
+
def __init__(self, model_name: str = "mock-model-v1"):
print(f"Initializing Mock Model: {model_name}...")
self.model_name = model_name
@@ -39,7 +35,7 @@ def generate(self, messages: list, temperature: float = 0.7, max_tokens: int = 5
"""Simulate non-streaming generation."""
# Just pick a random response
return random.choice(self.responses)
-
+
def stream_generate(self, messages: list, temperature: float = 0.7, max_tokens: int = 512):
"""Simulate streaming generation by yielding words."""
response = random.choice(self.responses)
@@ -54,7 +50,7 @@ def get_embeddings(self, texts: Union[str, List[str]]) -> List[List[float]]:
"""Simulate embedding generation with random vectors."""
if isinstance(texts, str):
texts = [texts]
-
+
# Return random vectors of size 128
return [[random.uniform(-1, 1) for _ in range(128)] for _ in texts]
@@ -89,11 +85,11 @@ def chat_logic(payload: schemas.ChatCompletionRequest):
temperature=payload.temperature,
max_tokens=payload.max_tokens or 512
)
-
+
# Mock token counts
prompt_tokens = sum(len(m.content.split()) for m in payload.messages) * 2
completion_tokens = len(response_text.split()) * 2
-
+
return schemas.ChatCompletionResponse(
id=f"chatcmpl-{uuid.uuid4().hex[:8]}",
object="chat.completion",
diff --git a/test/llm/test_six_schemas.py b/test/llm/test_six_schemas.py
deleted file mode 100644
index c1dd566..0000000
--- a/test/llm/test_six_schemas.py
+++ /dev/null
@@ -1,399 +0,0 @@
-"""
-Tests for the six modality schemas added in #179, plus the FRICTION #15 fix
-in _BaseLLMMixin._wrap_llm_response.
-
-These tests do not require torch / transformers — they exercise the schema
-validation path and the mixin wrap path directly.
-"""
-
-import pytest
-from pydantic import ValidationError
-
-from apipod.common import schemas
-from apipod.engine.llm.base_llm_mixin import _BaseLLMMixin
-
-
-# ----------------------------------------------------------------------
-# Registry / supported tuple
-# ----------------------------------------------------------------------
-
-def test_supported_tuple_has_nine_entries():
- """SUPPORTED_LLM_REQUEST_SCHEMAS should now hold 3 original + 6 new = 9."""
- assert len(schemas.SUPPORTED_LLM_REQUEST_SCHEMAS) == 9
-
-
-def test_supported_tuple_contains_new_request_classes():
- for cls in (
- schemas.ImageGenerationRequest,
- schemas.VideoGenerationRequest,
- schemas.AudioRequest,
- schemas.Generation3DRequest,
- schemas.VisionRequest,
- schemas.MultimodalEmbeddingRequest,
- ):
- assert cls in schemas.SUPPORTED_LLM_REQUEST_SCHEMAS
-
-
-def test_mixin_configs_has_nine_entries():
- m = _BaseLLMMixin()
- assert len(m._llm_configs) == 9
-
-
-def test_mixin_configs_maps_new_requests_to_responses():
- m = _BaseLLMMixin()
- expected = {
- schemas.ImageGenerationRequest: (schemas.ImageGenerationResponse, "image_generation"),
- schemas.VideoGenerationRequest: (schemas.VideoGenerationResponse, "video_generation"),
- schemas.AudioRequest: (schemas.AudioResponse, "audio"),
- schemas.Generation3DRequest: (schemas.Generation3DResponse, "generation_3d"),
- schemas.VisionRequest: (schemas.VisionResponse, "vision"),
- schemas.MultimodalEmbeddingRequest: (schemas.MultimodalEmbeddingResponse, "embedding_multimodal"),
- }
- for req_cls, expected_cfg in expected.items():
- assert m._llm_configs[req_cls] == expected_cfg
-
-
-# ----------------------------------------------------------------------
-# Schema field defaults and required-field validation
-# ----------------------------------------------------------------------
-
-def test_image_generation_request_minimal_valid():
- r = schemas.ImageGenerationRequest(model="flux", prompt="a cat")
- assert r.model == "flux"
- assert r.prompt == "a cat"
- assert r.num_images == 1 # documented default
- assert r.image is None
- assert r.mask is None
-
-
-def test_image_generation_request_missing_prompt_raises():
- with pytest.raises(ValidationError):
- schemas.ImageGenerationRequest(model="flux")
-
-
-def test_video_generation_request_defaults():
- r = schemas.VideoGenerationRequest(model="hunyuan", prompt="x")
- assert r.duration_s == 5.0
- assert r.fps == 24
-
-
-def test_video_generation_request_missing_prompt_raises():
- with pytest.raises(ValidationError):
- schemas.VideoGenerationRequest(model="hunyuan")
-
-
-def test_audio_request_all_inputs_optional_except_model():
- r = schemas.AudioRequest(model="whisper")
- assert r.text is None
- assert r.audio is None
- assert r.voice is None
-
-
-def test_generation_3d_request_output_format_default():
- r = schemas.Generation3DRequest(model="triposr")
- assert r.output_format == "glb"
-
-
-def test_vision_request_requires_image():
- with pytest.raises(ValidationError):
- schemas.VisionRequest(model="clip")
-
-
-def test_multimodal_embedding_request_all_inputs_optional():
- r = schemas.MultimodalEmbeddingRequest(model="clip")
- assert r.input is None
- assert r.image is None
- assert r.audio is None
-
-
-# ----------------------------------------------------------------------
-# Response shape validation (Literal["..."] guards)
-# ----------------------------------------------------------------------
-
-def test_image_generation_response_object_literal():
- r = schemas.ImageGenerationResponse(
- id="imggen-1-abc",
- object="image_generation",
- created=1,
- model="flux",
- data=[schemas.ImageGenerationData(url="http://x/i.png")],
- )
- assert r.object == "image_generation"
-
-
-def test_image_generation_response_wrong_object_raises():
- with pytest.raises(ValidationError):
- schemas.ImageGenerationResponse(
- id="imggen-1-abc",
- object="wrong",
- created=1,
- model="flux",
- data=[],
- )
-
-
-def test_video_generation_response_object_literal():
- r = schemas.VideoGenerationResponse(
- id="vidgen-1-abc",
- object="video_generation",
- created=1,
- model="hunyuan",
- data=[schemas.VideoGenerationData(url="http://x/v.mp4")],
- )
- assert r.object == "video_generation"
-
-
-def test_video_generation_response_wrong_object_raises():
- with pytest.raises(ValidationError):
- schemas.VideoGenerationResponse(
- id="vidgen-1-abc",
- object="wrong",
- created=1,
- model="hunyuan",
- data=[],
- )
-
-
-def test_audio_response_object_literal():
- r = schemas.AudioResponse(
- id="aud-1-abc",
- object="audio",
- created=1,
- model="whisper",
- data=[schemas.AudioData(text="hola")],
- )
- assert r.object == "audio"
-
-
-def test_audio_response_wrong_object_raises():
- with pytest.raises(ValidationError):
- schemas.AudioResponse(
- id="aud-1-abc",
- object="wrong",
- created=1,
- model="whisper",
- data=[],
- )
-
-
-def test_generation_3d_response_object_literal():
- r = schemas.Generation3DResponse(
- id="gen3d-1-abc",
- object="generation_3d",
- created=1,
- model="triposr",
- data=[schemas.Generation3DData(url="http://x/c.glb")],
- )
- assert r.object == "generation_3d"
-
-
-def test_generation_3d_response_wrong_object_raises():
- with pytest.raises(ValidationError):
- schemas.Generation3DResponse(
- id="gen3d-1-abc",
- object="wrong",
- created=1,
- model="triposr",
- data=[],
- )
-
-
-def test_vision_response_object_literal():
- r = schemas.VisionResponse(
- id="vis-1-abc",
- object="vision",
- created=1,
- model="clip",
- data=[schemas.VisionData(labels=[])],
- )
- assert r.object == "vision"
-
-
-def test_vision_response_wrong_object_raises():
- with pytest.raises(ValidationError):
- schemas.VisionResponse(
- id="vis-1-abc",
- object="wrong",
- created=1,
- model="clip",
- data=[],
- )
-
-
-def test_multimodal_embedding_response_object_is_list():
- """Like EmbeddingResponse: outer object is 'list', each data item is 'embedding'."""
- r = schemas.MultimodalEmbeddingResponse(
- object="list",
- data=[
- schemas.MultimodalEmbeddingData(
- embedding=[0.1, 0.2],
- index=0,
- modality="text",
- )
- ],
- model="clip",
- )
- assert r.object == "list"
- assert r.data[0].object == "embedding" # default on the data item
-
-
-# ----------------------------------------------------------------------
-# Mixin wrap path — covers FRICTION #15 fix + each new endpoint_type
-# ----------------------------------------------------------------------
-
-@pytest.fixture
-def mixin():
- return _BaseLLMMixin()
-
-
-def test_friction_15_embedding_wrap_outer_object_is_list(mixin):
- """
- FRICTION #15 regression. Pre-fix this raised ValidationError because the
- mixin passed object='embedding' to EmbeddingResponse, which declares
- object: Literal['list']. After the fix the wrap path succeeds and the
- outer object is 'list'.
- """
- req = schemas.EmbeddingRequest(model="bge", input="hello")
- wrapped = mixin._wrap_llm_response(
- result={
- "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}],
- "usage": {"prompt_tokens": 1, "total_tokens": 1},
- },
- response_model=schemas.EmbeddingResponse,
- endpoint_type="embedding",
- openai_req=req,
- )
- assert wrapped.object == "list"
- assert wrapped.data[0].object == "embedding"
-
-
-def test_image_generation_wrap(mixin):
- req = schemas.ImageGenerationRequest(model="flux", prompt="cat")
- wrapped = mixin._wrap_llm_response(
- result={"data": [{"url": "http://x/i.png", "seed": 42}]},
- response_model=schemas.ImageGenerationResponse,
- endpoint_type="image_generation",
- openai_req=req,
- )
- assert wrapped.object == "image_generation"
- assert wrapped.data[0].url == "http://x/i.png"
- assert wrapped.id.startswith("imggen-")
-
-
-def test_video_generation_wrap(mixin):
- req = schemas.VideoGenerationRequest(model="hunyuan", prompt="x")
- wrapped = mixin._wrap_llm_response(
- result={"data": [{"url": "http://x/v.mp4", "duration_s": 5.0}]},
- response_model=schemas.VideoGenerationResponse,
- endpoint_type="video_generation",
- openai_req=req,
- )
- assert wrapped.object == "video_generation"
- assert wrapped.id.startswith("vidgen-")
-
-
-def test_audio_wrap(mixin):
- req = schemas.AudioRequest(model="whisper", text="hi")
- wrapped = mixin._wrap_llm_response(
- result={"data": [{"text": "hola", "language": "es"}]},
- response_model=schemas.AudioResponse,
- endpoint_type="audio",
- openai_req=req,
- )
- assert wrapped.object == "audio"
- assert wrapped.data[0].text == "hola"
-
-
-def test_generation_3d_wrap(mixin):
- req = schemas.Generation3DRequest(model="triposr", prompt="chair")
- wrapped = mixin._wrap_llm_response(
- result={"data": [{"url": "http://x/c.glb", "output_format": "glb"}]},
- response_model=schemas.Generation3DResponse,
- endpoint_type="generation_3d",
- openai_req=req,
- )
- assert wrapped.object == "generation_3d"
- assert wrapped.id.startswith("gen3d-")
-
-
-# 32x32 solid-red PNG (~100 bytes / ~140 b64 chars). The previous 1x1 PNG
-# was too short for media_toolkit's base64 sniffer (it bails on payloads
-# below the heuristic length to avoid false positives on short tokens).
-def _make_png_b64() -> str:
- import base64
- import struct
- import zlib
-
- w = h = 32
- ihdr = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)
- raw = b""
- for _ in range(h):
- raw += b"\x00" + bytes((255, 0, 0) * w)
- idat = zlib.compress(raw)
- png = (
- b"\x89PNG\r\n\x1a\n"
- + struct.pack(">I", len(ihdr)) + b"IHDR" + ihdr + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr))
- + struct.pack(">I", len(idat)) + b"IDAT" + idat + struct.pack(">I", zlib.crc32(b"IDAT" + idat))
- + struct.pack(">I", 0) + b"IEND" + struct.pack(">I", zlib.crc32(b"IEND"))
- )
- return base64.b64encode(png).decode("ascii")
-
-
-_TINY_PNG_B64 = _make_png_b64()
-
-
-def test_vision_wrap(mixin):
- """
- Vision branch in _wrap_llm_response had no test before; the elif path was
- never exercised. Covers the branch end-to-end.
- """
- req = schemas.VisionRequest(model="clip", image=_TINY_PNG_B64)
- wrapped = mixin._wrap_llm_response(
- result={
- "data": [
- {
- "labels": [{"label": "cat", "score": 0.95}],
- "text": None,
- }
- ]
- },
- response_model=schemas.VisionResponse,
- endpoint_type="vision",
- openai_req=req,
- )
- assert wrapped.object == "vision"
- assert wrapped.id.startswith("vis-")
- assert wrapped.data[0].labels[0].label == "cat"
- assert wrapped.data[0].labels[0].score == 0.95
-
-
-def test_multimodal_embedding_wrap(mixin):
- req = schemas.MultimodalEmbeddingRequest(model="clip", input="hi")
- wrapped = mixin._wrap_llm_response(
- result={
- "data": [
- {
- "object": "embedding",
- "embedding": [0.1, 0.2],
- "index": 0,
- "modality": "text",
- }
- ]
- },
- response_model=schemas.MultimodalEmbeddingResponse,
- endpoint_type="embedding_multimodal",
- openai_req=req,
- )
- assert wrapped.object == "list"
- assert wrapped.data[0].embedding == [0.1, 0.2]
-
-
-def test_unknown_endpoint_type_still_raises(mixin):
- req = schemas.ImageGenerationRequest(model="x", prompt="y")
- with pytest.raises(ValueError, match="Unknown endpoint type"):
- mixin._wrap_llm_response(
- result={"data": []},
- response_model=schemas.ImageGenerationResponse,
- endpoint_type="not_a_real_type",
- openai_req=req,
- )
From a780ccc95235b0d92e4d93296c55755155f6c67b Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Sun, 14 Jun 2026 11:53:29 +0200
Subject: [PATCH 10/38] Implemented response wrapping also for streaming
endpoints. No need to specify the fields of the LLM Chunks anymore
---
README.md | 5 +-
apipod/engine/backend/fastapi/router.py | 38 ++++++++++++---
apipod/engine/backend/schema_resolve.py | 63 +++++++++++++++++++++++++
docs/README_TECH.md | 2 +-
test/llm/test_llm_mock.py | 62 +++++-------------------
5 files changed, 111 insertions(+), 59 deletions(-)
diff --git a/README.md b/README.md
index 6d2b1f4..36d645e 100644
--- a/README.md
+++ b/README.md
@@ -97,7 +97,7 @@ def transcribe(audio: AudioFile):
APIPod ships request/response schemas for the common AI payloads: chat, completions, embeddings, image/video/3D generation, vision, transcription, text-to-speech and voice cloning. Annotate one parameter with a request schema and APIPod handles validation, media parsing, response wrapping and streaming. The wire format mirrors the OpenAI API, so OpenAI-compatible clients work out of the box. `model` is optional everywhere — your service usually *is* the model.
-**Chat** — return a plain string (or a dict / full response object) and get a complete `chat.completion` envelope. Return a generator and `"stream": true` requests receive server-sent events:
+**Chat** — return a plain string (or a dict / full response object) and get a complete `chat.completion` envelope. For `"stream": true` requests just yield raw tokens: APIPod wraps each into a `ChatCompletionChunk` server-sent event — generating the chunk `id`, `created` and `object` out of the box — and emits the closing chunk plus the `[DONE]` sentinel:
```python
from apipod import APIPod
@@ -107,6 +107,9 @@ app = APIPod()
@app.endpoint("/chat")
def chat(request: ChatCompletionRequest):
+ if request.stream:
+ # Yield plain tokens — APIPod wraps them into ChatCompletionChunk SSE events.
+ return my_llm.stream(request.messages, temperature=request.temperature)
answer = my_llm.generate(request.messages, temperature=request.temperature)
return answer # auto-wrapped into a ChatCompletionResponse
```
diff --git a/apipod/engine/backend/fastapi/router.py b/apipod/engine/backend/fastapi/router.py
index 5a5745a..87b0c33 100644
--- a/apipod/engine/backend/fastapi/router.py
+++ b/apipod/engine/backend/fastapi/router.py
@@ -20,7 +20,10 @@
from apipod.engine.backend.fastapi.exception_handling import _FastAPIExceptionHandler
from apipod.engine.backend.schema_resolve import (
SchemaBinding,
+ SchemaStreamSerializer,
+ SSE_DONE,
SSE_STREAM_TAGS,
+ STREAM_CHUNK_SPECS,
iter_media_chunks,
wrap_schema_response,
)
@@ -458,10 +461,12 @@ def _is_streaming_endpoint(self, func: Callable) -> bool:
def _as_streaming_response(self, result: Any, binding: SchemaBinding) -> StreamingResponse | None:
"""
- Build a :class:`StreamingResponse` if the endpoint result is streamable:
+ Build a :class:`StreamingResponse` if a schema endpoint result is streamable:
- ``AudioFile`` / ``VideoFile`` stream their encoded bytes as raw chunks
with the proper media content type (OpenAI ``stream_format="audio"``);
- - generators stream as-is: SSE for token-delta endpoints, raw bytes else.
+ - generators of raw tokens are wrapped into the schema's standardized chunk
+ SSE stream (e.g. ``ChatCompletionChunk``) when one is registered, else
+ streamed as-is: SSE for token-delta endpoints, raw bytes otherwise.
Returns ``None`` when the result cannot be streamed.
"""
if isinstance(result, (AudioFile, VideoFile)):
@@ -469,28 +474,47 @@ def _as_streaming_response(self, result: Any, binding: SchemaBinding) -> Streami
return StreamingResponse(iter_media_chunks(result), media_type=media_type)
if inspect.isgenerator(result) or inspect.isasyncgen(result):
+ if binding.tag in STREAM_CHUNK_SPECS:
+ generator = self._schema_chunk_stream(result, binding)
+ else:
+ generator = self._stream_generator(result)
media_type = "text/event-stream" if binding.tag in SSE_STREAM_TAGS else "application/octet-stream"
return StreamingResponse(
- self._stream_generator(result),
+ generator,
media_type=media_type,
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
return None
+ async def _schema_chunk_stream(self, result: Any, binding: SchemaBinding) -> AsyncIterator[str]:
+ """Wrap raw token deltas into the schema's standardized chunk SSE stream."""
+ serializer = SchemaStreamSerializer(binding)
+ async for token in self._stream_generator(result):
+ yield serializer.delta(token)
+ yield serializer.finish()
+ yield SSE_DONE
+
# ------------------------------------------------------------------
# Endpoint decorators (task / standard / streaming)
# ------------------------------------------------------------------
def _modify_result_decorator(self, func: Callable, plan: EndpointExecutionPlan) -> Callable:
"""
- Wraps responses of schema endpoints to their given Response format.
- Serializes media-files to and other objects to JSON
+ Wraps responses of schema endpoints into their declared response format.
+
+ Streaming results (a token generator or media file) are turned into the
+ schema's standardized SSE/byte stream; everything else is wrapped into the
+ response model and serialized (media files -> JSON).
"""
@functools.wraps(func)
def sync_wrapper(*args, **kwargs):
result = func(*args, **kwargs)
- if plan.schema_binding is not None:
- result = wrap_schema_response(result, plan.schema_binding)
+ binding = plan.schema_binding
+ if binding is not None:
+ streaming = self._as_streaming_response(result, binding)
+ if streaming is not None:
+ return streaming
+ result = wrap_schema_response(result, binding)
return JobResultFactory._serialize_result(result)
return sync_wrapper
diff --git a/apipod/engine/backend/schema_resolve.py b/apipod/engine/backend/schema_resolve.py
index 12a6d4b..e8821f0 100644
--- a/apipod/engine/backend/schema_resolve.py
+++ b/apipod/engine/backend/schema_resolve.py
@@ -20,11 +20,13 @@
"""
import inspect
+import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from types import UnionType
from typing import Any, Callable, Iterator, Optional, Type, Union, get_args, get_origin
+from pydantic import BaseModel
from media_toolkit import MediaFile
from apipod.common.schemas import *
@@ -233,3 +235,64 @@ def iter_media_chunks(media_file: MediaFile, chunk_size: int = 64 * 1024) -> Ite
buffer = media_file.to_bytes_io()
while chunk := buffer.read(chunk_size):
yield chunk
+
+
+# ----------------------------------------------------------------------------
+# Streaming: wrap raw token deltas into standardized chunk SSE events
+# ----------------------------------------------------------------------------
+
+SSE_DONE = "data: [DONE]\n\n"
+
+
+def _to_sse(chunk: BaseModel) -> str:
+ """Serialize a chunk model into a single server-sent-event data line."""
+ return f"data: {chunk.model_dump_json()}\n\n"
+
+
+def _build_chat_chunk(chunk_id: str, created: int, content: Optional[str], finish_reason: Optional[str]) -> ChatCompletionChunk:
+ return ChatCompletionChunk(
+ id=chunk_id,
+ created=created,
+ choices=[ChatStreamChoice(index=0, delta=ChatDelta(content=content), finish_reason=finish_reason)],
+ )
+
+
+@dataclass(frozen=True)
+class StreamChunkSpec:
+ """How raw token deltas of a streaming schema are wrapped into chunk events."""
+ id_prefix: str
+ build: Callable[[str, int, Optional[str], Optional[str]], BaseModel]
+
+
+# Schema tags whose token stream is wrapped into a standardized chunk SSE stream.
+# Endpoints for these tags may simply yield text tokens; APIPod owns the envelope.
+STREAM_CHUNK_SPECS: dict[str, StreamChunkSpec] = {
+ "chat": StreamChunkSpec(id_prefix="chatcmpl", build=_build_chat_chunk),
+}
+
+
+class SchemaStreamSerializer:
+ """
+ Turns the raw token deltas yielded by a streaming schema endpoint into its
+ standardized chunk SSE stream (e.g. ``ChatCompletionChunk``).
+
+ The endpoint only yields text tokens; APIPod owns the envelope: a stable
+ chunk id, the ``created`` timestamp and the ``object`` discriminator are
+ generated out of the box, closed by a final delta and the ``[DONE]`` sentinel.
+ """
+
+ def __init__(self, binding: SchemaBinding):
+ spec = STREAM_CHUNK_SPECS.get(binding.tag)
+ if spec is None:
+ raise ValueError(f"Schema '{binding.tag}' does not support streaming chunks.")
+ self._build = spec.build
+ self._chunk_id = f"{spec.id_prefix}-{uuid.uuid4().hex[:8]}"
+ self._created = int(datetime.now(timezone.utc).timestamp())
+
+ def delta(self, content: str) -> str:
+ """Serialize a content token into a chunk SSE event."""
+ return _to_sse(self._build(self._chunk_id, self._created, content, None))
+
+ def finish(self) -> str:
+ """Serialize the closing chunk (``finish_reason='stop'``)."""
+ return _to_sse(self._build(self._chunk_id, self._created, None, "stop"))
diff --git a/docs/README_TECH.md b/docs/README_TECH.md
index fa7b482..d3de165 100644
--- a/docs/README_TECH.md
+++ b/docs/README_TECH.md
@@ -107,7 +107,7 @@ Three streaming paths exist today:
1. **Generator endpoints** — any endpoint function that is a (async) generator, or whose return annotation declares an iterator, is served as an SSE `StreamingResponse` (FastAPI) or as a native generator handed back to RunPod (`return_aggregate_stream`). Detection is annotation/inspect-based — no source-code heuristics.
2. **Schema endpoints with `stream=true`** — the request schema carries the `stream` flag (chat, completion, transcription, speech, video generation). Streaming bypasses the queue; what gets streamed depends on what the function returns:
- - a **generator**: streamed as-is — SSE for token-delta endpoints (chat/completion/transcription, e.g. `ChatCompletionChunk` events), raw bytes otherwise;
+ - a **generator of raw tokens**: for tags with a registered chunk model (`STREAM_CHUNK_SPECS`, currently `chat`), `SchemaStreamSerializer` wraps each token into the standardized chunk model (`ChatCompletionChunk`) as an SSE event — APIPod generates the stable chunk `id`, the `created` timestamp and the `object` discriminator, then closes with a final delta and the `[DONE]` sentinel. The endpoint only yields text. Other token-delta tags without a chunk model stream their tokens as-is (SSE), and non-SSE tags stream raw bytes;
- an **`AudioFile`/`VideoFile`**: its encoded bytes are chunked into a `StreamingResponse` with the file's media content type — raw audio chunks, not SSE, matching OpenAI's `stream_format="audio"` behavior;
- anything else: the regular wrapped JSON response (the endpoint cannot stream).
On RunPod the same logic applies, but chunks pass through `_yield_native_stream`, which base64-encodes binary chunks because RunPod transports JSON.
diff --git a/test/llm/test_llm_mock.py b/test/llm/test_llm_mock.py
index fb09cfc..3e96e2c 100644
--- a/test/llm/test_llm_mock.py
+++ b/test/llm/test_llm_mock.py
@@ -6,7 +6,6 @@
from typing import List, Union
from datetime import datetime
import time
-import uuid
import random
from apipod import APIPod
@@ -91,7 +90,6 @@ def chat_logic(payload: schemas.ChatCompletionRequest):
completion_tokens = len(response_text.split()) * 2
return schemas.ChatCompletionResponse(
- id=f"chatcmpl-{uuid.uuid4().hex[:8]}",
object="chat.completion",
created=int(datetime.now().timestamp()),
model=payload.model or state.model.model_name,
@@ -153,55 +151,19 @@ def embeddings_logic(payload: schemas.EmbeddingRequest):
@app.endpoint(path="/chat")
def chat_endpoint(payload: schemas.ChatCompletionRequest):
"""Chat completion endpoint with streaming support."""
+ if state.model is None:
+ raise RuntimeError("Model not initialized")
+
if payload.stream:
- def sse_generator():
- if state.model is None:
- state.model = MockModel("mock-model-v1")
-
- chunk_id = f"chatcmpl-{uuid.uuid4().hex[:8]}"
- created_time = int(datetime.now().timestamp())
- model_name = payload.model or state.model.model_name
-
- for token in state.model.stream_generate(
- messages=payload.messages,
- temperature=payload.temperature,
- max_tokens=payload.max_tokens
- ):
- chunk = schemas.ChatCompletionChunk(
- id=chunk_id,
- object="chat.completion.chunk",
- created=created_time,
- model=model_name,
- choices=[
- schemas.ChatStreamChoice(
- index=0,
- delta=schemas.ChatDelta(content=token),
- finish_reason=None
- )
- ]
- )
- yield f"data: {chunk.model_dump_json()}\n\n"
-
- # Final chunk
- final_chunk = schemas.ChatCompletionChunk(
- id=chunk_id,
- object="chat.completion.chunk",
- created=created_time,
- model=model_name,
- choices=[
- schemas.ChatStreamChoice(
- index=0,
- delta=schemas.ChatDelta(content=None),
- finish_reason="stop"
- )
- ]
- )
- yield f"data: {final_chunk.model_dump_json()}\n\n"
- yield "data: [DONE]\n\n"
-
- return sse_generator()
- else:
- return chat_logic(payload)
+ # Just yield raw tokens — APIPod wraps each into a ChatCompletionChunk SSE
+ # event and emits the closing chunk + [DONE] sentinel out of the box.
+ return state.model.stream_generate(
+ messages=payload.messages,
+ temperature=payload.temperature,
+ max_tokens=payload.max_tokens,
+ )
+
+ return chat_logic(payload)
@app.endpoint(path="/embeddings")
def embeddings_endpoint(payload: schemas.EmbeddingRequest):
From bf551e4981cce4862b6ad938db6a010d949c3c32 Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Sun, 14 Jun 2026 16:41:28 +0200
Subject: [PATCH 11/38] updated the runpod router. Layed out the
streaming_store for local apipod mimicking. Implented streaming capabilities
for media files. Implemented auto-chunking.
---
apipod/__init__.py | 4 +
apipod/api.py | 16 +-
apipod/engine/backend/fastapi/router.py | 284 +++---------------
.../engine/backend/fastapi/streaming_mixin.py | 171 +++++++++++
apipod/engine/backend/runpod/router.py | 158 +++++-----
apipod/engine/backend/schema_resolve.py | 9 +-
apipod/engine/endpoint_config.py | 107 ++++---
apipod/engine/jobs/base_job.py | 1 +
apipod/engine/jobs/job_result.py | 2 +
apipod/engine/queue/job_queue.py | 57 +++-
apipod/engine/streaming/__init__.py | 21 ++
apipod/engine/streaming/local_stream_store.py | 142 +++++++++
apipod/engine/streaming/stream_producer.py | 33 ++
apipod/engine/streaming/stream_serializer.py | 156 ++++++++++
apipod/engine/streaming/stream_store.py | 110 +++++++
docs/README_TECH.md | 17 +-
.../test_clients_six_schemas.py | 0
.../test_service_six_schemas.py | 0
test/test_streaming.py | 188 ++++++++++++
19 files changed, 1122 insertions(+), 354 deletions(-)
create mode 100644 apipod/engine/backend/fastapi/streaming_mixin.py
create mode 100644 apipod/engine/streaming/__init__.py
create mode 100644 apipod/engine/streaming/local_stream_store.py
create mode 100644 apipod/engine/streaming/stream_producer.py
create mode 100644 apipod/engine/streaming/stream_serializer.py
create mode 100644 apipod/engine/streaming/stream_store.py
rename {examples => test}/test_clients_six_schemas.py (100%)
rename {examples => test}/test_service_six_schemas.py (100%)
create mode 100644 test/test_streaming.py
diff --git a/apipod/__init__.py b/apipod/__init__.py
index 402e0d9..6582334 100644
--- a/apipod/__init__.py
+++ b/apipod/__init__.py
@@ -2,6 +2,7 @@
from apipod.engine.jobs.base_job import BaseJob, LocalJob
from apipod.engine.jobs.job_progress import JobProgress
from apipod.engine.jobs.job_result import FileModel, JobLinks, JobMetrics, JobResult
+from apipod.engine.streaming import StreamStore, LocalStreamStore, StreamProducer
from media_toolkit import MediaFile, ImageFile, AudioFile, VideoFile, MediaList, MediaDict
from apipod.common import constants
@@ -25,6 +26,9 @@
"JobLinks",
"JobMetrics",
"JobResult",
+ "StreamStore",
+ "LocalStreamStore",
+ "StreamProducer",
"MediaFile",
"ImageFile",
"AudioFile",
diff --git a/apipod/api.py b/apipod/api.py
index 2bd0b81..e95f726 100644
--- a/apipod/api.py
+++ b/apipod/api.py
@@ -54,9 +54,17 @@ def APIPod(
use_job_queue = True
job_queue = custom_job_queue
else:
- job_queue = _create_job_queue() if use_job_queue else None
+ from apipod.engine.queue.job_queue import JobQueue
+ job_queue = JobQueue() if use_job_queue else None
if backend_class == SocaityFastAPIRouter:
+ # Streaming (serverless emulation): when a queue runs, the worker relays
+ # streaming output into a stream store consumed via GET /stream/{job_id}.
+ # Default to the in-memory LocalStreamStore; a deployment can inject its
+ # own (e.g. Redis-backed) store. Plain FastAPI (no queue) streams directly
+ # and uses no stream store.
+ if use_job_queue and "stream_store" not in kwargs:
+ kwargs["stream_store"] = _create_stream_store()
return backend_class(job_queue=job_queue, *args, **kwargs)
else:
return backend_class(*args, **kwargs)
@@ -146,3 +154,9 @@ def _create_job_queue() -> JobQueueInterface:
from apipod.engine.queue.job_queue import JobQueue
return JobQueue()
+
+
+def _create_stream_store():
+ from apipod.engine.streaming.local_stream_store import LocalStreamStore
+
+ return LocalStreamStore()
diff --git a/apipod/engine/backend/fastapi/router.py b/apipod/engine/backend/fastapi/router.py
index 87b0c33..b9859e9 100644
--- a/apipod/engine/backend/fastapi/router.py
+++ b/apipod/engine/backend/fastapi/router.py
@@ -3,34 +3,26 @@
import threading
import logging
from contextlib import asynccontextmanager
-from collections.abc import AsyncIterator, Iterator
-from typing import Any, Union, Callable, get_origin, get_type_hints
-from fastapi import APIRouter, FastAPI, Request, Response, status
+from typing import Union, Callable
+from fastapi import APIRouter, FastAPI, Response, status
from fastapi.exceptions import HTTPException
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.responses import JSONResponse
from apipod.common.settings import APIPOD_PORT, APIPOD_HOST
from apipod.common.constants import SERVER_HEALTH
from apipod.engine.jobs.job_result import JobResultFactory, JobResult
-from apipod.engine.endpoint_config import FastApiEndpointConfigurator, EndpointExecutionPlan
+from apipod.engine.endpoint_config import build_plan, EndpointExecutionPlan
+from apipod.engine.streaming.stream_serializer import build_stream_producer
from apipod.engine.base_backend import _BaseBackend
from apipod.engine.queue.queue_mixin import _QueueMixin
from apipod.engine.backend.fastapi.file_handling_mixin import _fast_api_file_handling_mixin
+from apipod.engine.backend.fastapi.streaming_mixin import _FastAPIStreamingMixin
from apipod.engine.utils import normalize_name
from apipod.engine.backend.fastapi.exception_handling import _FastAPIExceptionHandler
-from apipod.engine.backend.schema_resolve import (
- SchemaBinding,
- SchemaStreamSerializer,
- SSE_DONE,
- SSE_STREAM_TAGS,
- STREAM_CHUNK_SPECS,
- iter_media_chunks,
- wrap_schema_response,
-)
-from media_toolkit import AudioFile, VideoFile
-
-
-class SocaityFastAPIRouter(APIRouter, _BaseBackend, _QueueMixin, _fast_api_file_handling_mixin, _FastAPIExceptionHandler):
+from apipod.engine.backend.schema_resolve import wrap_schema_response
+
+
+class SocaityFastAPIRouter(APIRouter, _BaseBackend, _QueueMixin, _fast_api_file_handling_mixin, _FastAPIStreamingMixin, _FastAPIExceptionHandler):
"""
FastAPI router extension that adds support for task endpoints.
@@ -108,9 +100,15 @@ def __init__(
self.prefix = prefix
self.stream_store = stream_store
self.gateway_stream_url_prefix = gateway_stream_url_prefix
- self.add_standard_routes()
- self._endpoint_configurator = FastApiEndpointConfigurator(self)
+ # Let the queue produce streaming job output into the stream store so a
+ # client can consume it via GET /stream/{job_id} (serverless emulation).
+ if self.job_queue is not None and self.stream_store is not None:
+ set_store = getattr(self.job_queue, "set_stream_store", None)
+ if callable(set_store):
+ set_store(self.stream_store)
+
+ self.add_standard_routes()
# Exception handling
_FastAPIExceptionHandler.__init__(self)
@@ -264,47 +262,6 @@ def post_cancel_job(self, job_id: str) -> dict:
) from None
return {"id": job_id, "status": "cancelled", "message": "Job cancelled."}
- async def stream_job_sse(self, job_id: str, request: Request):
- """Server-Sent Events for streaming job output (requires stream_store)."""
- job_id = job_id.strip().strip('"').strip("'").strip("?").strip("#")
- if self.stream_store is None:
- raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Streaming not configured.")
- if self.job_queue is None:
- raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Job queue not configured.")
-
- jq = self.job_queue
- job_data = jq.get_job_status(job_id) if hasattr(jq, "get_job_status") else None
-
- if job_data is None:
- raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Job '{job_id}' not found.")
-
- st = (job_data.get("status") or "").lower()
- if st != "streaming" and not self.stream_store.stream_exists(job_id):
- raise HTTPException(
- status_code=status.HTTP_409_CONFLICT,
- detail=f"Job '{job_id}' is not streaming (status: {job_data.get('status')}).",
- )
-
- async def _event_generator():
- try:
- async for chunk in self.stream_store.read_chunks(job_id):
- if await request.is_disconnected():
- break
- yield chunk
- except Exception:
- self._logger.exception("Error during stream delivery | job_id=%s", job_id)
- yield 'data: {"error": "Internal stream error"}\n\n'
-
- return StreamingResponse(
- _event_generator(),
- media_type="text/event-stream",
- headers={
- "Cache-Control": "no-cache",
- "Connection": "keep-alive",
- "X-Accel-Buffering": "no",
- },
- )
-
def endpoint(self, path: str, methods: list[str] | None = None, max_upload_file_size_mb: int = None, queue_size: int = 500, use_queue: bool = None, *args, **kwargs):
"""
Unified endpoint decorator.
@@ -325,18 +282,22 @@ def endpoint(self, path: str, methods: list[str] | None = None, max_upload_file_
should_use_queue = self._determine_queue_usage(use_queue, normalized_path)
def decorator(func: Callable) -> Callable:
- plan = self._create_endpoint_plan(
- func=func,
- normalized_path=normalized_path,
+ plan = build_plan(
+ func,
+ path=normalized_path,
methods=methods,
max_upload_file_size_mb=max_upload_file_size_mb,
queue_size=queue_size,
should_use_queue=should_use_queue,
route_args=args,
- route_kwargs=kwargs
+ route_kwargs=kwargs,
)
- if plan.is_streaming:
+ # Streaming with a queue + stream store mimics a real deployment:
+ # the job is queued, the worker produces chunks into the stream store
+ # and the client consumes them from GET /stream/{job_id}. Without a
+ # queue (plain FastAPI) streaming goes straight to the client.
+ if plan.is_streaming and not plan.should_use_queue:
return self._create_streaming_endpoint_decorator(plan)(func)
if plan.should_use_queue:
return self._create_task_endpoint_decorator(plan)(func)
@@ -345,85 +306,6 @@ def decorator(func: Callable) -> Callable:
return decorator
- def _create_endpoint_plan(
- self,
- *,
- func: Callable,
- normalized_path: str,
- methods: list[str] | None,
- max_upload_file_size_mb: int | None,
- queue_size: int,
- should_use_queue: bool,
- route_args: tuple,
- route_kwargs: dict,
- ) -> EndpointExecutionPlan:
- """Build an orchestration plan for endpoint registration and execution."""
- return self._endpoint_configurator.build_plan(
- func=func,
- path=normalized_path,
- methods=methods,
- max_upload_file_size_mb=max_upload_file_size_mb,
- queue_size=queue_size,
- should_use_queue=should_use_queue,
- route_args=route_args,
- route_kwargs=route_kwargs,
- )
-
- async def _execute_func(self, func, **kwargs):
- """
- Execute a function, handling both sync and async functions.
-
- Args:
- func: Function to execute
- **kwargs: Arguments to pass to function
-
- Returns:
- Result of function execution
- """
- if inspect.iscoroutinefunction(func):
- return await func(**kwargs)
- else:
- # Sync function - run in thread pool to avoid blocking
- import asyncio
- loop = asyncio.get_event_loop()
- return await loop.run_in_executor(None, functools.partial(func, **kwargs))
-
- async def _stream_generator(self, result):
- """
- Convert a sync or async generator into an async generator for StreamingResponse.
-
- Args:
- result: Generator (sync or async) that yields streaming chunks
-
- Yields:
- Streaming chunks as strings or bytes
- """
- import asyncio
-
- if inspect.isasyncgen(result):
- # Async generator - yield directly
- async for chunk in result:
- if isinstance(chunk, (str, bytes)):
- yield chunk
- else:
- yield str(chunk)
- elif inspect.isgenerator(result):
- # Sync generator - run in executor
- loop = asyncio.get_event_loop()
- try:
- while True:
- chunk = await loop.run_in_executor(None, next, result, StopIteration)
- if chunk is StopIteration:
- break
- if isinstance(chunk, (str, bytes)):
- yield chunk
- else:
- yield str(chunk)
- except StopIteration:
- pass
- else:
- raise TypeError(f"Expected generator, got {type(result)}")
-
def _normalize_endpoint_path(self, path: str) -> str:
"""Normalize the endpoint path to ensure it starts with '/'."""
normalized = normalize_name(path, preserve_paths=True)
@@ -439,81 +321,32 @@ def _determine_queue_usage(self, use_queue: bool = None, path: str = None) -> bo
return self.job_queue is not None
- def _is_streaming_endpoint(self, func: Callable) -> bool:
- """
- Determine whether the function streams its output: it is a (async)
- generator function, or its return annotation declares an iterator.
- """
- target = inspect.unwrap(func)
- if inspect.isgeneratorfunction(target) or inspect.isasyncgenfunction(target):
- return True
-
- try:
- return_type = get_type_hints(target).get('return')
- except Exception:
- return False
-
- if return_type is None:
- return False
-
- origin = get_origin(return_type) or return_type
- return inspect.isclass(origin) and issubclass(origin, (Iterator, AsyncIterator))
-
- def _as_streaming_response(self, result: Any, binding: SchemaBinding) -> StreamingResponse | None:
- """
- Build a :class:`StreamingResponse` if a schema endpoint result is streamable:
- - ``AudioFile`` / ``VideoFile`` stream their encoded bytes as raw chunks
- with the proper media content type (OpenAI ``stream_format="audio"``);
- - generators of raw tokens are wrapped into the schema's standardized chunk
- SSE stream (e.g. ``ChatCompletionChunk``) when one is registered, else
- streamed as-is: SSE for token-delta endpoints, raw bytes otherwise.
- Returns ``None`` when the result cannot be streamed.
- """
- if isinstance(result, (AudioFile, VideoFile)):
- media_type = getattr(result, "content_type", None) or "application/octet-stream"
- return StreamingResponse(iter_media_chunks(result), media_type=media_type)
-
- if inspect.isgenerator(result) or inspect.isasyncgen(result):
- if binding.tag in STREAM_CHUNK_SPECS:
- generator = self._schema_chunk_stream(result, binding)
- else:
- generator = self._stream_generator(result)
- media_type = "text/event-stream" if binding.tag in SSE_STREAM_TAGS else "application/octet-stream"
- return StreamingResponse(
- generator,
- media_type=media_type,
- headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
- )
-
- return None
-
- async def _schema_chunk_stream(self, result: Any, binding: SchemaBinding) -> AsyncIterator[str]:
- """Wrap raw token deltas into the schema's standardized chunk SSE stream."""
- serializer = SchemaStreamSerializer(binding)
- async for token in self._stream_generator(result):
- yield serializer.delta(token)
- yield serializer.finish()
- yield SSE_DONE
-
# ------------------------------------------------------------------
# Endpoint decorators (task / standard / streaming)
# ------------------------------------------------------------------
- def _modify_result_decorator(self, func: Callable, plan: EndpointExecutionPlan) -> Callable:
+ def _modify_result_decorator(self, func: Callable, plan: EndpointExecutionPlan, *, queued: bool) -> Callable:
"""
- Wraps responses of schema endpoints into their declared response format.
+ Wraps endpoint responses into their final transport form.
+
+ Streamable results (a token/chunk generator) become a stream:
+ - ``queued`` (job queue): a :class:`StreamProducer` is returned so the
+ worker relays chunks into the stream store (consumed via /stream) while
+ aggregating the full result for /status;
+ - direct (no queue): a :class:`StreamingResponse` is returned to the client.
- Streaming results (a token generator or media file) are turned into the
- schema's standardized SSE/byte stream; everything else is wrapped into the
- response model and serialized (media files -> JSON).
+ Non-streaming schema results are wrapped into the response model; all
+ results are then serialized (media files -> JSON).
"""
@functools.wraps(func)
def sync_wrapper(*args, **kwargs):
result = func(*args, **kwargs)
+
+ producer = build_stream_producer(result, plan.schema_binding)
+ if producer is not None:
+ return producer if queued else self._streaming_response_from_producer(producer)
+
binding = plan.schema_binding
if binding is not None:
- streaming = self._as_streaming_response(result, binding)
- if streaming is not None:
- return streaming
result = wrap_schema_response(result, binding)
return JobResultFactory._serialize_result(result)
return sync_wrapper
@@ -540,7 +373,7 @@ def _create_task_endpoint_decorator(self, plan: EndpointExecutionPlan):
)
def decorator(func: Callable) -> Callable:
- func = self._modify_result_decorator(func, plan)
+ func = self._modify_result_decorator(func, plan, queued=True)
# Add job queue functionality and prepare for FastAPI file handling
queue_decorated = queue_decorator(func)
# Register the function so workers can execute it (dev mode).
@@ -565,41 +398,12 @@ def _create_standard_endpoint_decorator(self, plan: EndpointExecutionPlan):
)
def decorator(func: Callable) -> Callable:
- result_modified = self._modify_result_decorator(func, plan)
+ result_modified = self._modify_result_decorator(func, plan, queued=False)
with_file_upload_signature = self._prepare_func_for_media_file_upload_with_fastapi(result_modified, plan.max_upload_file_size_mb)
return fastapi_route_decorator(with_file_upload_signature)
return decorator
- def _create_streaming_endpoint_decorator(self, plan: EndpointExecutionPlan):
- """Create a decorator for streaming endpoints."""
- kwargs = dict(plan.route_kwargs)
- custom_headers = kwargs.pop('response_headers', None)
-
- fastapi_route_decorator = self.api_route(
- path=plan.path,
- methods=plan.active_methods,
- *plan.route_args,
- **kwargs
- )
-
- def decorator(func: Callable) -> Callable:
- @functools.wraps(func)
- async def streaming_wrapper(*w_args, **w_kwargs):
- result = await self._execute_func(func, *w_args, **w_kwargs)
- generator = self._stream_generator(result)
-
- headers = {"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}
- if custom_headers:
- headers.update(custom_headers)
-
- return StreamingResponse(generator, media_type="text/event-stream", headers=headers)
-
- with_file_upload_signature = self._prepare_func_for_media_file_upload_with_fastapi(streaming_wrapper, plan.max_upload_file_size_mb)
- return fastapi_route_decorator(with_file_upload_signature)
-
- return decorator
-
def get(self, path: str = None, queue_size: int = 100, *args, **kwargs):
"""
Create a GET endpoint.
diff --git a/apipod/engine/backend/fastapi/streaming_mixin.py b/apipod/engine/backend/fastapi/streaming_mixin.py
new file mode 100644
index 0000000..24642d8
--- /dev/null
+++ b/apipod/engine/backend/fastapi/streaming_mixin.py
@@ -0,0 +1,171 @@
+"""
+FastAPI-specific streaming transport, factored out of the router.
+
+This mixin owns the FastAPI-side mechanics for turning a streaming endpoint
+result into an HTTP response:
+
+- ``_execute_func`` — run a sync or async function without blocking the event loop;
+- ``_stream_generator`` — adapt a sync/async generator into an async generator
+ for :class:`StreamingResponse` (direct non-queued path);
+- ``_streaming_response_from_producer`` — build a :class:`StreamingResponse` from
+ a :class:`StreamProducer` (direct non-queued path);
+- ``_create_streaming_endpoint_decorator`` — register a plain generator endpoint
+ directly on the FastAPI router (direct non-queued path);
+- ``stream_job_sse`` — SSE consumer route ``GET /stream/{job_id}`` that replays
+ a queued job's chunks from the stream store.
+
+Serialization details (encoding, aggregation, producer construction) live in
+:mod:`apipod.engine.streaming.stream_serializer`; endpoint introspection and
+plan building live in :mod:`apipod.engine.endpoint_config`.
+
+The mixin carries no state and needs no ``__init__``.
+"""
+
+import asyncio
+import functools
+import inspect
+from typing import Callable
+
+from fastapi import Request, status
+from fastapi.exceptions import HTTPException
+from fastapi.responses import StreamingResponse
+
+from apipod.engine.endpoint_config import EndpointExecutionPlan
+from apipod.engine.streaming.stream_producer import StreamProducer
+
+
+# Sentinel marking the end of a synchronous iterator drained via run_in_executor.
+_STREAM_END = object()
+
+
+class _FastAPIStreamingMixin:
+ """FastAPI streaming transport helpers mixed into the FastAPI router."""
+
+ # ------------------------------------------------------------------
+ # Execution helpers
+ # ------------------------------------------------------------------
+
+ async def _execute_func(self, func, **kwargs):
+ """Execute a function, handling both sync and async callables."""
+ if inspect.iscoroutinefunction(func):
+ return await func(**kwargs)
+ loop = asyncio.get_event_loop()
+ return await loop.run_in_executor(None, functools.partial(func, **kwargs))
+
+ async def _stream_generator(self, result):
+ """Adapt a sync or async generator into an async generator for StreamingResponse."""
+ if inspect.isasyncgen(result):
+ async for chunk in result:
+ yield chunk if isinstance(chunk, (str, bytes)) else str(chunk)
+ elif inspect.isgenerator(result):
+ loop = asyncio.get_event_loop()
+ while True:
+ chunk = await loop.run_in_executor(None, next, result, _STREAM_END)
+ if chunk is _STREAM_END:
+ break
+ yield chunk if isinstance(chunk, (str, bytes)) else str(chunk)
+ else:
+ raise TypeError(f"Expected generator, got {type(result)}")
+
+ # ------------------------------------------------------------------
+ # Direct streaming (non-queued)
+ # ------------------------------------------------------------------
+
+ def _streaming_response_from_producer(self, producer: StreamProducer) -> StreamingResponse:
+ """Build a direct :class:`StreamingResponse` from a producer (non-queued path)."""
+ async def _event_generator():
+ loop = asyncio.get_event_loop()
+ iterator = iter(producer.raw_chunks)
+ while True:
+ item = await loop.run_in_executor(None, next, iterator, _STREAM_END)
+ if item is _STREAM_END:
+ break
+ yield producer.to_chunk(item)
+ for closing_chunk in producer.closing:
+ yield closing_chunk
+
+ return StreamingResponse(
+ _event_generator(),
+ media_type=producer.media_type,
+ headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+ )
+
+ def _create_streaming_endpoint_decorator(self, plan: EndpointExecutionPlan):
+ """Register a plain generator endpoint served directly to the client (no queue)."""
+ kwargs = dict(plan.route_kwargs)
+ custom_headers = kwargs.pop("response_headers", None)
+
+ fastapi_route_decorator = self.api_route(
+ path=plan.path,
+ methods=plan.active_methods,
+ *plan.route_args,
+ **kwargs,
+ )
+
+ def decorator(func: Callable) -> Callable:
+ @functools.wraps(func)
+ async def streaming_wrapper(*w_args, **w_kwargs):
+ result = await self._execute_func(func, *w_args, **w_kwargs)
+ generator = self._stream_generator(result)
+
+ headers = {"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}
+ if custom_headers:
+ headers.update(custom_headers)
+
+ return StreamingResponse(generator, media_type="text/event-stream", headers=headers)
+
+ with_upload = self._prepare_func_for_media_file_upload_with_fastapi(
+ streaming_wrapper, plan.max_upload_file_size_mb
+ )
+ return fastapi_route_decorator(with_upload)
+
+ return decorator
+
+ # ------------------------------------------------------------------
+ # SSE consumer route (queued jobs)
+ # ------------------------------------------------------------------
+
+ async def stream_job_sse(self, job_id: str, request: Request):
+ """Server-Sent Events consumer for a queued streaming job (requires stream_store)."""
+ job_id = job_id.strip().strip('"').strip("'").strip("?").strip("#")
+ if self.stream_store is None:
+ raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Streaming not configured.")
+ if self.job_queue is None:
+ raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Job queue not configured.")
+
+ jq = self.job_queue
+ job_data = jq.get_job_status(job_id) if hasattr(jq, "get_job_status") else None
+ stream_open = self.stream_store.stream_exists(job_id)
+
+ # Serve the stream even when the job record is already gone (completed +
+ # cleaned up) as long as the stream is still draining.
+ if job_data is None and not stream_open:
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Job '{job_id}' not found.")
+
+ # Allow any not-yet-finished state; read_chunks waits for the producer.
+ st = ((job_data.get("status") if job_data else "") or "").lower()
+ if not stream_open and st not in {"pending", "processing", "streaming"}:
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail=f"Job '{job_id}' is not streaming (status: {st or 'unknown'}).",
+ )
+
+ async def _event_generator():
+ try:
+ async for chunk in self.stream_store.read_chunks(job_id):
+ if await request.is_disconnected():
+ break
+ yield chunk
+ except Exception:
+ self._logger.exception("Error during stream delivery | job_id=%s", job_id)
+ yield 'data: {"error": "Internal stream error"}\n\n'
+
+ return StreamingResponse(
+ _event_generator(),
+ media_type="text/event-stream",
+ headers={
+ "Cache-Control": "no-cache",
+ "Connection": "keep-alive",
+ "X-Accel-Buffering": "no",
+ },
+ )
diff --git a/apipod/engine/backend/runpod/router.py b/apipod/engine/backend/runpod/router.py
index 6ed2b37..33b6392 100644
--- a/apipod/engine/backend/runpod/router.py
+++ b/apipod/engine/backend/runpod/router.py
@@ -1,24 +1,35 @@
import asyncio
-import base64
import functools
import inspect
import traceback
from datetime import datetime, timezone
-from typing import Union, Callable
+from typing import Union, Callable, Iterator
from apipod.common import constants
from apipod.engine.jobs.base_job import JOB_STATUS
from apipod.engine.jobs.job_progress import JobProgressRunpod, JobProgress
-from apipod.engine.jobs.job_result import JobResultFactory, JobResult
+from apipod.engine.jobs.job_result import (
+ JobResultFactory,
+ JobResult,
+ JobMetrics,
+ _compute_duration_s,
+ _job_status_to_public,
+)
from apipod.engine.base_backend import _BaseBackend
+from apipod.engine.endpoint_config import build_plan, EndpointExecutionPlan
from apipod.engine.files.base_file_mixin import _BaseFileHandlingMixin
from apipod.engine.backend.schema_resolve import (
+ SchemaBinding,
+ SchemaStreamSerializer,
+ STREAM_CHUNK_SPECS,
iter_media_chunks,
+ prepare_schema_call,
wrap_schema_response,
)
+from apipod.engine.streaming.stream_serializer import as_sync_iter, encode_chunk, is_streaming_result
from apipod.engine.utils import normalize_name
-from apipod.common.settings import APIPOD_PROVIDER, APIPOD_PORT, DEFAULT_DATE_TIME_FORMAT
+from apipod.common.settings import APIPOD_PROVIDER, APIPOD_PORT
from media_toolkit import AudioFile, VideoFile
@@ -41,62 +52,75 @@ def endpoint(self, path: str = None, use_queue: bool = None, *args, **kwargs):
path = normalize_name(path, preserve_paths=True).strip("/")
def decorator(func: Callable) -> Callable:
- # binding = get_schema_binding(func)
+ plan = build_plan(func, path=path)
+ route = self._build_route(func, plan)
+ self.routes[path] = route
+ return route
+ return decorator
- @functools.wraps(func)
- def wrapper(*w_args, **w_kwargs):
- self.status = constants.SERVER_HEALTH.BUSY
- try:
- # if binding is None:
- # return self._execute_sync_or_async(func, w_args, w_kwargs)
+ def _build_route(self, func: Callable, plan: EndpointExecutionPlan) -> Callable:
+ """
+ Compose the callable registered for a route from its execution plan.
- # request = prepare_schema_call(binding, w_kwargs)
- result = self._execute_sync_or_async(func, w_args, w_kwargs)
+ Schema endpoints validate + media-parse their request themselves; plain
+ endpoints get the generic media file-upload conversion wrapped in.
+ """
+ @functools.wraps(func)
+ def wrapper(*w_args, **w_kwargs):
+ self.status = constants.SERVER_HEALTH.BUSY
+ try:
+ if plan.is_schema_endpoint:
+ return self._handle_schema_request(func, plan.schema_binding, w_args, w_kwargs)
+ result = self._execute_sync_or_async(func, w_args, w_kwargs)
+ # Plain generator endpoints stream too: turn the generator into a
+ # RunPod-native stream of JSON-safe chunks (RunPod aggregates it).
+ native_stream = self._as_native_stream(result)
+ return native_stream if native_stream is not None else result
+ finally:
+ self.status = constants.SERVER_HEALTH.RUNNING
- if getattr(request, "stream", False):
- if isinstance(result, (AudioFile, VideoFile)):
- return self._yield_native_stream(iter_media_chunks(result))
- if inspect.isgenerator(result) or inspect.isasyncgen(result):
- return self._yield_native_stream(result)
+ return wrapper if plan.is_schema_endpoint else self._handle_file_uploads(wrapper)
- return wrap_schema_response(result, binding)
- finally:
- self.status = constants.SERVER_HEALTH.RUNNING
+ # ------------------------------------------------------------------
+ # Standardized schema endpoints
+ # ------------------------------------------------------------------
+ def _handle_schema_request(self, func: Callable, binding: SchemaBinding, args: tuple, kwargs: dict):
+ """
+ Run a standardized schema endpoint: validate + media-parse the request
+ (``prepare_schema_call``), then stream or wrap the response into the
+ registered response model (``wrap_schema_response``).
+ """
+ request = prepare_schema_call(binding, kwargs)
+ result = self._execute_sync_or_async(func, args, kwargs)
- self.routes[path] = wrapper
- return wrapper
- return decorator
+ if getattr(request, "stream", False):
+ native_stream = self._as_native_stream(result, binding)
+ if native_stream is not None:
+ return native_stream
- @staticmethod
- def _encode_stream_chunk(chunk) -> str:
- """RunPod transports everything as JSON: binary chunks are base64-encoded."""
- if isinstance(chunk, bytes):
- return base64.b64encode(chunk).decode("ascii")
- return chunk if isinstance(chunk, str) else str(chunk)
+ return wrap_schema_response(result, binding)
- def _yield_native_stream(self, result):
- """Bridge a stream result (sync/async generator or StreamingResponse) into RunPod's native stream."""
- from starlette.responses import StreamingResponse
+ def _as_native_stream(self, result, binding: Union[SchemaBinding, None] = None) -> Union[Iterator, None]:
+ """
+ Wrap a streamable result into a RunPod-native generator of JSON-safe chunks.
- # If it's a StreamingResponse (shouldn't happen in RunPod, but handle it)
- if isinstance(result, StreamingResponse):
- result = result.body_iterator
+ - ``AudioFile`` / ``VideoFile`` → base64-encoded byte chunks;
+ - schema generator with a registered chunk model (e.g. chat) → standardized
+ ``ChatCompletionChunk`` stream via :class:`SchemaStreamSerializer`;
+ - any other generator → ``encode_chunk`` (base64 for binary, str for text).
- if inspect.isasyncgen(result):
- while True:
- try:
- yield self._encode_stream_chunk(self._run_in_loop(result.__anext__()))
- except StopAsyncIteration:
- break
- return
+ Returns ``None`` when the result is not streamable.
+ """
+ if isinstance(result, (AudioFile, VideoFile)):
+ return (encode_chunk(chunk) for chunk in iter_media_chunks(result))
- if inspect.isgenerator(result) or hasattr(result, "__iter__"):
- for chunk in result:
- yield self._encode_stream_chunk(chunk)
- return
+ if is_streaming_result(result):
+ tokens = as_sync_iter(result)
+ if binding is not None and binding.tag in STREAM_CHUNK_SPECS:
+ return SchemaStreamSerializer(binding).stream(tokens)
+ return (encode_chunk(chunk) for chunk in tokens)
- # Not a generator - shouldn't happen for streaming
- raise TypeError(f"Expected generator for streaming, got {type(result)}")
+ return None
def _execute_sync_or_async(self, func, args, kwargs):
if inspect.iscoroutinefunction(func):
@@ -172,15 +196,8 @@ def _router(self, path, job, **kwargs):
if missing_args:
raise Exception(f"Arguments {missing_args} are missing")
- # Handle file uploads and conversions
- route_function = self._handle_file_uploads(route_function)
-
- # Prepare result tracking
start_time = datetime.now(timezone.utc)
- result = JobResult(
- job_id=job["id"],
- created_at=start_time.strftime(DEFAULT_DATE_TIME_FORMAT),
- )
+ result = JobResult(job_id=job["id"], endpoint=path)
try:
# Execute the function (Sync or Async Handling)
@@ -196,33 +213,40 @@ def _router(self, path, job, **kwargs):
res = JobResultFactory._serialize_result(res)
result.result = res
- result.status = JOB_STATUS.FINISHED.value
+ result.status = _job_status_to_public(JOB_STATUS.FINISHED)
except Exception as e:
result.error = str(e)
- result.status = JOB_STATUS.FAILED.value
+ result.status = _job_status_to_public(JOB_STATUS.FAILED)
print(f"Job {job['id']} failed: {str(e)}")
traceback.print_exc()
- finally:
- result.updated_at = datetime.now(timezone.utc).strftime(DEFAULT_DATE_TIME_FORMAT)
- result = result.model_dump_json()
- return result
+ for arg in kwargs.values():
+ if isinstance(arg, JobProgressRunpod):
+ result.progress = float(arg._progress)
+ result.message = arg._message
+ break
+
+ inference_time_s = _compute_duration_s(start_time, datetime.now(timezone.utc))
+ if inference_time_s is not None:
+ result.metrics = JobMetrics(inference_time_s=inference_time_s)
+
+ return result.model_dump_json(exclude_none=True)
def _execute_route_function(self, route_function, kwargs):
"""
Execute a route function, handling both sync and async functions.
-
+
Args:
route_function: The function to execute
kwargs: Keyword arguments to pass to the function
-
+
Returns:
The result of the function execution
"""
if not inspect.iscoroutinefunction(route_function):
# Synchronous function - simple execution
return route_function(**kwargs)
-
+
# Async function - need event loop handling
try:
loop = asyncio.get_event_loop()
@@ -459,4 +483,4 @@ def start(self, port: int = APIPOD_PORT, provider: Union[constants.PROVIDER, str
self.start_runpod_serverless_localhost(port=port)
else:
import runpod.serverless
- runpod.serverless.start({"handler": self.handler, "return_aggregate_stream": True})
+ runpod.serverless.start({"handler": self.handler, "return_aggregate_stream": True})
\ No newline at end of file
diff --git a/apipod/engine/backend/schema_resolve.py b/apipod/engine/backend/schema_resolve.py
index e8821f0..5f7d4e6 100644
--- a/apipod/engine/backend/schema_resolve.py
+++ b/apipod/engine/backend/schema_resolve.py
@@ -24,7 +24,7 @@
from dataclasses import dataclass
from datetime import datetime, timezone
from types import UnionType
-from typing import Any, Callable, Iterator, Optional, Type, Union, get_args, get_origin
+from typing import Any, Callable, Iterable, Iterator, Optional, Type, Union, get_args, get_origin
from pydantic import BaseModel
from media_toolkit import MediaFile
@@ -296,3 +296,10 @@ def delta(self, content: str) -> str:
def finish(self) -> str:
"""Serialize the closing chunk (``finish_reason='stop'``)."""
return _to_sse(self._build(self._chunk_id, self._created, None, "stop"))
+
+ def stream(self, tokens: Iterable[str]) -> Iterator[str]:
+ """Wrap a synchronous token iterable into the full chunk SSE stream (deltas, closing chunk, [DONE])."""
+ for token in tokens:
+ yield self.delta(token)
+ yield self.finish()
+ yield SSE_DONE
diff --git a/apipod/engine/endpoint_config.py b/apipod/engine/endpoint_config.py
index 225a626..db10bac 100644
--- a/apipod/engine/endpoint_config.py
+++ b/apipod/engine/endpoint_config.py
@@ -1,5 +1,17 @@
-from dataclasses import dataclass
-from typing import Any, Callable
+"""
+Endpoint planning: backend-neutral analysis of an endpoint function.
+
+``is_streaming_endpoint`` inspects a function's definition to decide whether
+it streams its output. ``build_plan`` combines that with schema detection and
+optional backend-specific parameters (queue, upload limits, …) into an
+immutable :class:`EndpointExecutionPlan` — the shared contract used by every
+backend (FastAPI, RunPod).
+"""
+
+import inspect
+from collections.abc import AsyncIterator, Iterator
+from dataclasses import dataclass, field
+from typing import Any, Callable, get_origin, get_type_hints
from apipod.engine.backend.schema_resolve import SchemaBinding, get_schema_binding
@@ -10,16 +22,17 @@ class EndpointExecutionPlan:
Immutable plan describing how an endpoint should be registered and executed.
The plan separates endpoint configuration decisions from router registration
- mechanics. This keeps endpoint decorators compact and easier to reason about.
+ mechanics. Fields that only apply to a specific backend (e.g. queue options)
+ default to sensible no-ops so every backend can build a plan with what it needs.
"""
path: str
- methods: list[str] | None
- should_use_queue: bool
- max_upload_file_size_mb: int | None
- queue_size: int
- route_args: tuple[Any, ...]
- route_kwargs: dict[str, Any]
+ methods: list[str] | None = None
+ should_use_queue: bool = False
+ max_upload_file_size_mb: int | None = None
+ queue_size: int = 500
+ route_args: tuple[Any, ...] = ()
+ route_kwargs: dict[str, Any] = field(default_factory=dict)
schema_binding: SchemaBinding | None = None
is_streaming: bool = False
@@ -32,40 +45,52 @@ def active_methods(self) -> list[str]:
return ["POST"] if self.methods is None else self.methods
-class FastApiEndpointConfigurator:
- """
- Builds endpoint execution plans for the FastAPI backend.
+def is_streaming_endpoint(func: Callable) -> bool:
+ """Backend-neutral: True if *func* is a generator or annotated as an Iterator.
- This component configures endpoint behavior (schema dispatch, streaming,
- queue) independent from provider mechanics (FastAPI routing).
+ A generator function (``yield``) or async generator function (``async yield``)
+ is always considered streaming. A regular function whose return annotation
+ is ``Iterator[...]`` / ``AsyncIterator[...]`` or any subclass is also detected.
"""
+ target = inspect.unwrap(func)
+ if inspect.isgeneratorfunction(target) or inspect.isasyncgenfunction(target):
+ return True
+ try:
+ return_type = get_type_hints(target).get("return")
+ except Exception:
+ return False
+ if return_type is None:
+ return False
+ origin = get_origin(return_type) or return_type
+ return inspect.isclass(origin) and issubclass(origin, (Iterator, AsyncIterator))
- def __init__(self, router):
- self._router = router
- def build_plan(
- self,
- *,
- func: Callable,
- path: str,
- methods: list[str] | None,
- max_upload_file_size_mb: int | None,
- queue_size: int,
- should_use_queue: bool,
- route_args: tuple[Any, ...],
- route_kwargs: dict[str, Any],
- ) -> EndpointExecutionPlan:
- schema_binding = get_schema_binding(func)
- is_streaming = bool(schema_binding is None and self._router._is_streaming_endpoint(func))
+def build_plan(
+ func: Callable,
+ path: str,
+ *,
+ methods: list[str] | None = None,
+ should_use_queue: bool = False,
+ max_upload_file_size_mb: int | None = None,
+ queue_size: int = 500,
+ route_args: tuple[Any, ...] = (),
+ route_kwargs: dict[str, Any] | None = None,
+) -> EndpointExecutionPlan:
+ """Build a backend-neutral :class:`EndpointExecutionPlan` by inspecting *func*.
- return EndpointExecutionPlan(
- path=path,
- methods=methods,
- should_use_queue=should_use_queue,
- max_upload_file_size_mb=max_upload_file_size_mb,
- queue_size=queue_size,
- route_args=route_args,
- route_kwargs=route_kwargs,
- schema_binding=schema_binding,
- is_streaming=is_streaming,
- )
+ Both the FastAPI and RunPod backends call this. Backend-specific parameters
+ (``methods``, ``queue_*``, ``route_*``) are optional; they default to no-ops
+ so RunPod can simply call ``build_plan(func, path=path)``.
+ """
+ schema_binding = get_schema_binding(func)
+ return EndpointExecutionPlan(
+ path=path,
+ methods=methods,
+ should_use_queue=should_use_queue,
+ max_upload_file_size_mb=max_upload_file_size_mb,
+ queue_size=queue_size,
+ route_args=route_args,
+ route_kwargs=route_kwargs if route_kwargs is not None else {},
+ schema_binding=schema_binding,
+ is_streaming=schema_binding is None and is_streaming_endpoint(func),
+ )
diff --git a/apipod/engine/jobs/base_job.py b/apipod/engine/jobs/base_job.py
index c4e20e1..655f861 100644
--- a/apipod/engine/jobs/base_job.py
+++ b/apipod/engine/jobs/base_job.py
@@ -9,6 +9,7 @@
class JOB_STATUS(Enum):
QUEUED = "Queued"
PROCESSING = "Processing"
+ STREAMING = "Streaming"
FINISHED = "Finished"
FAILED = "Failed"
TIMEOUT = "Timeout"
diff --git a/apipod/engine/jobs/job_result.py b/apipod/engine/jobs/job_result.py
index b380970..8523f90 100644
--- a/apipod/engine/jobs/job_result.py
+++ b/apipod/engine/jobs/job_result.py
@@ -20,6 +20,7 @@ def _job_status_to_public(status: Any) -> Optional[str]:
return {
JOB_STATUS.QUEUED: "pending",
JOB_STATUS.PROCESSING: "processing",
+ JOB_STATUS.STREAMING: "streaming",
JOB_STATUS.FINISHED: "completed",
JOB_STATUS.FAILED: "failed",
JOB_STATUS.TIMEOUT: "failed",
@@ -30,6 +31,7 @@ def _job_status_to_public(status: Any) -> Optional[str]:
"queued": "pending",
"pending": "pending",
"processing": "processing",
+ "streaming": "streaming",
"finished": "completed",
"completed": "completed",
"failed": "failed",
diff --git a/apipod/engine/queue/job_queue.py b/apipod/engine/queue/job_queue.py
index 5aa6f0c..1628d49 100644
--- a/apipod/engine/queue/job_queue.py
+++ b/apipod/engine/queue/job_queue.py
@@ -6,7 +6,9 @@
from apipod.engine.queue.job_store import JobStore
from apipod.engine.jobs.base_job import BaseJob, LocalJob, JOB_STATUS
+from apipod.engine.jobs.job_result import _job_status_to_public
from apipod.engine.queue.job_queue_interface import JobQueueInterface
+from apipod.engine.streaming.stream_producer import StreamProducer
import inspect
T = TypeVar('T', bound=BaseJob)
@@ -35,6 +37,21 @@ def __init__(self, delete_orphan_jobs_after_s: int = 60 * 30):
self._shutdown = threading.Event()
self._job_threads: Dict[str, threading.Thread] = {}
self._delete_orphan_jobs_after_seconds = delete_orphan_jobs_after_s
+ # Optional StreamStore. When set, generator/streaming jobs produce their
+ # chunks into the store (so a client can consume GET /stream/{job_id})
+ # while still aggregating the full result for GET /status/{job_id}.
+ self._stream_store = None
+
+ def set_stream_store(self, stream_store) -> None:
+ """Attach a :class:`StreamStore` used to relay streaming job output."""
+ self._stream_store = stream_store
+
+ def get_job_status(self, job_id: str) -> Optional[dict]:
+ """Lightweight status lookup that never removes the job (used by /stream)."""
+ job = self.job_store.get_job(job_id)
+ if job is None:
+ return None
+ return {"id": job_id, "status": _job_status_to_public(job.status)}
def set_queue_size(self, job_function: callable, queue_size: int = 500) -> None:
self.queue_sizes[job_function.__name__] = queue_size
@@ -93,7 +110,15 @@ def _process_job(self, job: T) -> None:
self._inject_job_progress(job)
- job.result = job.job_function(**job.job_params)
+ result = job.job_function(**job.job_params)
+
+ # Streaming endpoints return a StreamProducer instead of a value: the
+ # worker relays chunks into the stream store (consumed via /stream)
+ # and keeps the aggregated result for /status.
+ if isinstance(result, StreamProducer):
+ result = self._run_stream(job, result)
+
+ job.result = result
job.job_progress.set_status(1.0)
self._complete_job(job=job, final_state=JOB_STATUS.FINISHED)
@@ -106,6 +131,36 @@ def _process_job(self, job: T) -> None:
print(f"Job {job.id} failed: {str(e)}")
traceback.print_exc() # Writes full traceback to stderr
+ def _run_stream(self, job: T, producer: StreamProducer):
+ """Drive a :class:`StreamProducer`: relay chunks into the stream store
+ (if configured) while collecting raw items to build the full result.
+
+ Without a stream store the chunks are simply aggregated, so the job still
+ returns a complete result via /status (streaming silently degrades).
+ """
+ store = self._stream_store
+ collected = []
+
+ if store is None:
+ for item in producer.raw_chunks:
+ collected.append(item)
+ return producer.aggregate(collected)
+
+ job.status = JOB_STATUS.STREAMING
+ store.open_stream(job.id)
+ try:
+ for item in producer.raw_chunks:
+ collected.append(item)
+ store.write_chunk(job.id, producer.to_chunk(item))
+ for closing_chunk in producer.closing:
+ store.write_chunk(job.id, closing_chunk)
+ store.close_stream(job.id)
+ except Exception as e:
+ store.close_stream(job.id, error=str(e))
+ raise
+
+ return producer.aggregate(collected)
+
def _complete_job(self, job: T, final_state: JOB_STATUS) -> T:
self.job_store.complete_job(job.id)
# setting status here, because if this is done earlier, race conditions in get_job are the problem
diff --git a/apipod/engine/streaming/__init__.py b/apipod/engine/streaming/__init__.py
new file mode 100644
index 0000000..3c3ab69
--- /dev/null
+++ b/apipod/engine/streaming/__init__.py
@@ -0,0 +1,21 @@
+from apipod.engine.streaming.stream_store import StreamStore
+from apipod.engine.streaming.local_stream_store import LocalStreamStore
+from apipod.engine.streaming.stream_producer import StreamProducer
+from apipod.engine.streaming.stream_serializer import (
+ as_sync_iter,
+ build_stream_producer,
+ encode_chunk,
+ is_streaming_result,
+ store_chunk,
+)
+
+__all__ = [
+ "StreamStore",
+ "LocalStreamStore",
+ "StreamProducer",
+ "as_sync_iter",
+ "build_stream_producer",
+ "encode_chunk",
+ "is_streaming_result",
+ "store_chunk",
+]
diff --git a/apipod/engine/streaming/local_stream_store.py b/apipod/engine/streaming/local_stream_store.py
new file mode 100644
index 0000000..5a8ec03
--- /dev/null
+++ b/apipod/engine/streaming/local_stream_store.py
@@ -0,0 +1,142 @@
+"""
+Adapter: LocalStreamStore
+
+In-process implementation of the :class:`StreamStore` port. This is the default
+backend APIPod uses on localhost to emulate how streaming behaves once a service
+is deployed (e.g. on Socaity, where a Redis-backed store relays chunks from the
+worker to the gateway).
+
+Everything lives in process memory, guarded by a lock:
+ - the in-process worker thread is the producer (``open_stream`` /
+ ``write_chunk`` / ``close_stream``);
+ - the FastAPI ``GET /stream/{job_id}`` route is the consumer
+ (``read_chunks``).
+
+It is intentionally simple — no external dependencies, no durability. A real
+deployment swaps in a distributed store; the endpoint code stays the same.
+"""
+
+import asyncio
+import logging
+import threading
+import time
+from typing import AsyncIterator, Dict, List, Optional
+
+from apipod.engine.streaming.stream_store import StreamStore
+
+logger = logging.getLogger(__name__)
+
+
+class _LocalStream:
+ """State for a single in-memory stream."""
+
+ def __init__(self, ttl_seconds: int):
+ self.chunks: List[str] = []
+ self.closed: bool = False
+ self.error: Optional[str] = None
+ self.expires_at: float = time.monotonic() + ttl_seconds
+
+
+class LocalStreamStore(StreamStore):
+ """Thread-safe, in-memory :class:`StreamStore` for local development/testing."""
+
+ def __init__(self, poll_interval_s: float = 0.02):
+ self._streams: Dict[str, _LocalStream] = {}
+ self._lock = threading.Lock()
+ # How often the async consumer polls the in-memory buffer for new data.
+ self._poll_interval_s = poll_interval_s
+
+ # -- Producer API ---------------------------------------------------
+
+ def open_stream(self, job_id: str, ttl_seconds: int = 3600) -> None:
+ with self._lock:
+ self._streams[job_id] = _LocalStream(ttl_seconds)
+ logger.debug("Stream opened | job_id=%s ttl=%ss", job_id, ttl_seconds)
+
+ def write_chunk(self, job_id: str, payload: str) -> None:
+ with self._lock:
+ stream = self._streams.get(job_id)
+ if stream is None:
+ # Be forgiving: auto-open if the producer forgot to.
+ stream = _LocalStream(ttl_seconds=3600)
+ self._streams[job_id] = stream
+ stream.chunks.append(payload)
+
+ def close_stream(self, job_id: str, *, error: Optional[str] = None) -> None:
+ with self._lock:
+ stream = self._streams.get(job_id)
+ if stream is None:
+ stream = _LocalStream(ttl_seconds=300)
+ self._streams[job_id] = stream
+ stream.error = error
+ stream.closed = True
+ logger.debug("Stream closed | job_id=%s error=%s", job_id, error)
+
+ # -- Consumer API ---------------------------------------------------
+
+ async def read_chunks(
+ self,
+ job_id: str,
+ *,
+ block_ms: int = 5000,
+ batch_size: int = 50,
+ ) -> AsyncIterator[str]:
+ # The consumer may connect before the producer has opened the stream
+ # (the worker picks the job up slightly after the HTTP response). Wait a
+ # bounded time for the stream to appear before giving up.
+ if not await self._await_stream(job_id, timeout_s=max(block_ms / 1000.0, 5.0)):
+ return
+
+ index = 0
+ idle_s = 0.0
+ keepalive_after_s = block_ms / 1000.0
+
+ while True:
+ with self._lock:
+ stream = self._streams.get(job_id)
+ if stream is None:
+ return
+ new_chunks = stream.chunks[index:index + batch_size]
+ index += len(new_chunks)
+ closed = stream.closed
+ error = stream.error
+ drained = index >= len(stream.chunks)
+
+ if new_chunks:
+ idle_s = 0.0
+ for chunk in new_chunks:
+ yield chunk
+ continue
+
+ if closed and drained:
+ if error:
+ yield f'data: {{"error": {error!r}}}\n\n'
+ self.delete_stream(job_id)
+ return
+
+ await asyncio.sleep(self._poll_interval_s)
+ idle_s += self._poll_interval_s
+ if idle_s >= keepalive_after_s:
+ idle_s = 0.0
+ # SSE comment keeps proxies / browsers from timing out.
+ yield ": keepalive\n\n"
+
+ async def _await_stream(self, job_id: str, timeout_s: float) -> bool:
+ deadline = time.monotonic() + timeout_s
+ while time.monotonic() < deadline:
+ with self._lock:
+ if job_id in self._streams:
+ return True
+ await asyncio.sleep(self._poll_interval_s)
+ return job_id in self._streams
+
+ # -- Lifecycle ------------------------------------------------------
+
+ def delete_stream(self, job_id: str) -> None:
+ with self._lock:
+ self._streams.pop(job_id, None)
+ logger.debug("Stream deleted | job_id=%s", job_id)
+
+ def stream_exists(self, job_id: str) -> bool:
+ with self._lock:
+ return job_id in self._streams
diff --git a/apipod/engine/streaming/stream_producer.py b/apipod/engine/streaming/stream_producer.py
new file mode 100644
index 0000000..0ab8880
--- /dev/null
+++ b/apipod/engine/streaming/stream_producer.py
@@ -0,0 +1,33 @@
+"""
+StreamProducer: the bridge between a streaming endpoint and the stream store.
+
+When a streaming endpoint runs under a job queue, its result cannot be sent to
+the client directly (the HTTP response already returned a ``JobResult``).
+Instead the worker produces the stream into a :class:`StreamStore` while the
+client consumes it from ``GET /stream/{job_id}``.
+
+The router knows *how* to serialize a given result (a ChatCompletion token
+stream, encoded media bytes, or a plain generator); the queue worker only knows
+the lifecycle. ``StreamProducer`` carries that router-built knowledge to the
+worker without leaking schema details into the queue:
+
+ - ``raw_chunks`` – the user generator's raw items (tokens / bytes / objects)
+ - ``to_chunk`` – serialize one raw item into a store/SSE chunk (str)
+ - ``closing`` – chunks appended after the stream (e.g. finish + ``[DONE]``)
+ - ``media_type`` – content type for the direct (non-queued) StreamingResponse
+ - ``aggregate`` – build the full ``/status`` result from the raw items, so a
+ client polling status (instead of streaming) gets the
+ complete result once the job finishes.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Iterator, List
+
+
+@dataclass
+class StreamProducer:
+ raw_chunks: Iterator[Any]
+ to_chunk: Callable[[Any], str]
+ aggregate: Callable[[List[Any]], Any]
+ closing: List[str] = field(default_factory=list)
+ media_type: str = "text/event-stream"
diff --git a/apipod/engine/streaming/stream_serializer.py b/apipod/engine/streaming/stream_serializer.py
new file mode 100644
index 0000000..6874f20
--- /dev/null
+++ b/apipod/engine/streaming/stream_serializer.py
@@ -0,0 +1,156 @@
+"""
+Backend-neutral stream serialization shared by all transport adapters.
+
+This module is the single place that knows how to:
+ - detect a streaming runtime result (``is_streaming_result``);
+ - bridge an async generator into a sync iterator (``as_sync_iter``);
+ - encode raw chunks into JSON-safe strings (``encode_chunk`` for RunPod,
+ ``store_chunk`` for the FastAPI stream store);
+ - aggregate all raw chunks back into a full result for ``/status`` polling;
+ - assemble a :class:`StreamProducer` for the FastAPI queued/direct path.
+
+Both :class:`SocaityFastAPIRouter` and :class:`SocaityRunpodRouter` import
+from here instead of each maintaining their own copy.
+"""
+
+import asyncio
+import base64
+import inspect
+from typing import Any, Iterator, Optional
+
+from media_toolkit import MediaFile
+
+from apipod.engine.backend.schema_resolve import (
+ SchemaBinding,
+ SchemaStreamSerializer,
+ SSE_DONE,
+ SSE_STREAM_TAGS,
+ STREAM_CHUNK_SPECS,
+ wrap_schema_response,
+)
+from apipod.engine.jobs.job_result import JobResultFactory
+from apipod.engine.streaming.stream_producer import StreamProducer
+
+
+# ---------------------------------------------------------------------------
+# Detection
+# ---------------------------------------------------------------------------
+
+def is_streaming_result(result: Any) -> bool:
+ """Return True if *result* is a sync or async generator."""
+ return inspect.isgenerator(result) or inspect.isasyncgen(result)
+
+
+# ---------------------------------------------------------------------------
+# Sync/async bridging
+# ---------------------------------------------------------------------------
+
+def as_sync_iter(result: Any) -> Iterator:
+ """Return a synchronous iterator over a sync or async generator."""
+ return _drain_async_gen(result) if inspect.isasyncgen(result) else result
+
+
+def _drain_async_gen(agen) -> Iterator:
+ """Consume an async generator synchronously (for worker / sync contexts)."""
+ loop = asyncio.new_event_loop()
+ try:
+ while True:
+ try:
+ yield loop.run_until_complete(agen.__anext__())
+ except StopAsyncIteration:
+ return
+ finally:
+ loop.close()
+
+
+# ---------------------------------------------------------------------------
+# Chunk encoding
+# ---------------------------------------------------------------------------
+
+def _to_base64(chunk: Any) -> Optional[str]:
+ """Return a base64 string when *chunk* is bytes / bytearray / MediaFile; else None."""
+ if isinstance(chunk, MediaFile):
+ chunk = chunk.to_bytes()
+ if isinstance(chunk, (bytes, bytearray)):
+ return base64.b64encode(bytes(chunk)).decode("ascii")
+ return None
+
+
+def encode_chunk(chunk: Any) -> str:
+ """JSON-safe encoding without SSE framing (for RunPod transport).
+
+ Binary / MediaFile → base64 string; str → unchanged; other → str().
+ """
+ b64 = _to_base64(chunk)
+ return b64 if b64 is not None else (chunk if isinstance(chunk, str) else str(chunk))
+
+
+def store_chunk(chunk: Any) -> str:
+ """SSE-framed encoding for the FastAPI stream store.
+
+ Binary / MediaFile → ``data: {base64}\\n\\n``; str → unchanged (passthrough);
+ other → ``data: {chunk}\\n\\n``.
+ """
+ b64 = _to_base64(chunk)
+ if b64 is not None:
+ return f"data: {b64}\n\n"
+ return chunk if isinstance(chunk, str) else f"data: {chunk}\n\n"
+
+
+# ---------------------------------------------------------------------------
+# Aggregation (full /status result from streamed items)
+# ---------------------------------------------------------------------------
+
+def aggregate_plain(items: list) -> Any:
+ """Join text chunks or serialize each item for the /status result."""
+ if items and all(isinstance(i, str) for i in items):
+ return "".join(items)
+ return [JobResultFactory._serialize_result(i) for i in items]
+
+
+def aggregate_schema_tokens(items: list, binding: SchemaBinding) -> Any:
+ """Reconstruct the schema response from streamed token items."""
+ return JobResultFactory._serialize_result(
+ wrap_schema_response("".join(str(i) for i in items), binding)
+ )
+
+
+# ---------------------------------------------------------------------------
+# StreamProducer construction (FastAPI queued + direct path)
+# ---------------------------------------------------------------------------
+
+def build_stream_producer(result: Any, binding: Optional[SchemaBinding]) -> Optional[StreamProducer]:
+ """
+ Build a :class:`StreamProducer` from a streamable endpoint result, or
+ return ``None`` when the result is not a generator.
+
+ Schema endpoints with a registered chunk model (e.g. chat) wrap tokens
+ into the standardized ``ChatCompletionChunk`` SSE stream; all other
+ generators produce SSE-framed text / base64-encoded binary chunks.
+ """
+ if not is_streaming_result(result):
+ return None
+
+ raw_chunks = as_sync_iter(result)
+
+ if binding is not None and binding.tag in STREAM_CHUNK_SPECS:
+ serializer = SchemaStreamSerializer(binding)
+ return StreamProducer(
+ raw_chunks=raw_chunks,
+ to_chunk=serializer.delta,
+ closing=[serializer.finish(), SSE_DONE],
+ media_type="text/event-stream",
+ aggregate=lambda items, b=binding: aggregate_schema_tokens(items, b),
+ )
+
+ media_type = (
+ "text/event-stream"
+ if binding is None or binding.tag in SSE_STREAM_TAGS
+ else "application/octet-stream"
+ )
+ return StreamProducer(
+ raw_chunks=raw_chunks,
+ to_chunk=store_chunk,
+ media_type=media_type,
+ aggregate=aggregate_plain,
+ )
diff --git a/apipod/engine/streaming/stream_store.py b/apipod/engine/streaming/stream_store.py
new file mode 100644
index 0000000..fa0feb0
--- /dev/null
+++ b/apipod/engine/streaming/stream_store.py
@@ -0,0 +1,110 @@
+"""
+Port: StreamStore
+
+Abstract interface for reading and writing streaming data chunks associated
+with a job. Implementations may keep the chunks in process memory (the local
+test backend), in Redis Streams, in Kafka, or in any other append-only log.
+
+In a real deployment the producer and the consumer live in different
+processes:
+
+ - The **worker** (producer) calls :meth:`open_stream` / :meth:`write_chunk`
+ as result chunks (ChatCompletion deltas, encoded ``AudioFile`` /
+ ``VideoFile`` bytes, or chunks of any generator endpoint) are produced.
+ - The **gateway** (consumer) calls :meth:`read_chunks` to relay those
+ chunks to the client over Server-Sent Events.
+ - Both sides use :meth:`close_stream` / :meth:`delete_stream` for lifecycle
+ management.
+
+APIPod on localhost emulates that split inside a single process: the in-process
+worker thread produces, while the FastAPI ``GET /stream/{job_id}`` route
+consumes. The default backend is :class:`LocalStreamStore`; deployments swap in
+their own implementation (e.g. a Redis-backed store) without touching the
+endpoint code.
+
+Design decisions:
+ - :meth:`read_chunks` is an **async generator** so the gateway can yield
+ chunks straight into a ``StreamingResponse`` without buffering.
+ - :meth:`write_chunk` / :meth:`open_stream` / :meth:`close_stream` are
+ **synchronous** because the worker writes from a (sync) worker context.
+ - :meth:`close_stream` signals completion (optionally with an error) so the
+ consumer's read loop exits cleanly.
+"""
+
+from abc import ABC, abstractmethod
+from typing import AsyncIterator, Optional
+
+
+class StreamStore(ABC):
+ """Port for append-only stream storage keyed by ``job_id``."""
+
+ # -- Producer API (worker side) -------------------------------------
+
+ @abstractmethod
+ def open_stream(self, job_id: str, ttl_seconds: int = 3600) -> None:
+ """
+ Prepare a stream for writing.
+
+ Implementations should create the underlying resource and set a
+ safety-net TTL to prevent leaks if neither side cleans up.
+ """
+ ...
+
+ @abstractmethod
+ def write_chunk(self, job_id: str, payload: str) -> None:
+ """
+ Append a single chunk (typically one SSE ``data:`` line) to the stream.
+
+ Must be safe to call from a synchronous (worker) context.
+ """
+ ...
+
+ @abstractmethod
+ def close_stream(self, job_id: str, *, error: Optional[str] = None) -> None:
+ """
+ Signal that the producer is done.
+
+ Writes a terminal marker so the consumer knows the stream is complete.
+ If *error* is provided, the marker carries the error payload.
+ """
+ ...
+
+ # -- Consumer API (gateway side) ------------------------------------
+
+ @abstractmethod
+ async def read_chunks(
+ self,
+ job_id: str,
+ *,
+ block_ms: int = 5000,
+ batch_size: int = 50,
+ ) -> AsyncIterator[str]:
+ """
+ Async generator that yields chunks until the terminal marker is read.
+
+ Parameters:
+ block_ms: how long to wait for new data before emitting a
+ keep-alive (used for timeout / disconnect checks).
+ batch_size: max chunks returned per internal read.
+ """
+ ...
+ # Make this an async generator at the type level.
+ if False: # pragma: no cover
+ yield ""
+
+ # -- Lifecycle ------------------------------------------------------
+
+ @abstractmethod
+ def delete_stream(self, job_id: str) -> None:
+ """
+ Remove the stream resource entirely.
+
+ Called once the consumer has read the final chunk, or by a periodic
+ cleanup for abandoned streams.
+ """
+ ...
+
+ @abstractmethod
+ def stream_exists(self, job_id: str) -> bool:
+ """Return ``True`` if the stream key exists."""
+ ...
diff --git a/docs/README_TECH.md b/docs/README_TECH.md
index d3de165..b8f1240 100644
--- a/docs/README_TECH.md
+++ b/docs/README_TECH.md
@@ -29,11 +29,12 @@ apipod/
│ ├── base_backend.py # Shared backend base (title, version, health)
│ ├── endpoint_config.py # EndpointExecutionPlan + configurator (how to register an endpoint)
│ ├── backend/
-│ │ ├── fastapi/ # SocaityFastAPIRouter + file handling, exceptions
+│ │ ├── fastapi/ # SocaityFastAPIRouter + file handling, streaming mixin, exceptions
│ │ └── runpod/ # SocaityRunpodRouter (serverless, path-based)
│ ├── files/ # _BaseFileHandlingMixin + parse_schema_media_fields (inputs → MediaFile)
│ ├── jobs/ # BaseJob, JobProgress, JobResult (+ factory)
│ ├── queue/ # JobQueue, JobStore, _QueueMixin (enqueue instead of block)
+│ ├── streaming/ # StreamStore port + LocalStreamStore + StreamProducer
│ └── signatures/ # Signature inspection policies (media params, Body vs Form, …)
└── deploy/ # `apipod build`: Dockerfile generation, dependency/CUDA detection
```
@@ -110,8 +111,18 @@ Three streaming paths exist today:
- a **generator of raw tokens**: for tags with a registered chunk model (`STREAM_CHUNK_SPECS`, currently `chat`), `SchemaStreamSerializer` wraps each token into the standardized chunk model (`ChatCompletionChunk`) as an SSE event — APIPod generates the stable chunk `id`, the `created` timestamp and the `object` discriminator, then closes with a final delta and the `[DONE]` sentinel. The endpoint only yields text. Other token-delta tags without a chunk model stream their tokens as-is (SSE), and non-SSE tags stream raw bytes;
- an **`AudioFile`/`VideoFile`**: its encoded bytes are chunked into a `StreamingResponse` with the file's media content type — raw audio chunks, not SSE, matching OpenAI's `stream_format="audio"` behavior;
- anything else: the regular wrapped JSON response (the endpoint cannot stream).
- On RunPod the same logic applies, but chunks pass through `_yield_native_stream`, which base64-encodes binary chunks because RunPod transports JSON.
-3. **Job streaming** — `GET /stream/{job_id}` replays chunks from a pluggable stream store while a queued job is in `streaming` state (gateway integration).
+ On RunPod the same logic applies (`_as_native_stream`): the request is validated and media-parsed by `prepare_schema_call`, the response is wrapped by `wrap_schema_response`, and stream chunks are base64-encoded when binary because RunPod transports JSON.
+3. **Job streaming (serverless emulation)** — when a queue is configured, a streaming endpoint does **not** stream on the request connection. The job is enqueued and the response returns a `JobResult` immediately (with a `stream` link). The worker (producer) then relays the chunks into a **stream store**, and the client (consumer) reads them from `GET /stream/{job_id}` as SSE. A client that prefers to wait can poll `GET /status/{job_id}` instead and receive the **full aggregated result** once the job finishes. This mirrors a real deployment, where producer (worker) and consumer (gateway) live in different processes.
+
+### The stream store
+
+The **stream store** is the pluggable backend that buffers a job's chunks between the worker and the gateway. Its key idea is to decouple *producing* a stream (sync, worker side) from *consuming* it (async SSE, gateway side), so the same endpoint code streams identically whether it runs on localhost or on a real platform.
+
+- **`StreamStore`** (`engine/streaming/stream_store.py`) is the port (abstract base class). Producer methods (`open_stream`, `write_chunk`, `close_stream`) are synchronous because the worker writes from a sync context; the consumer method `read_chunks` is an async generator that yields straight into a `StreamingResponse`. `delete_stream` / `stream_exists` cover lifecycle.
+- **`LocalStreamStore`** (`engine/streaming/local_stream_store.py`) is the default in-memory implementation APIPod uses on localhost — thread-safe, no external dependencies. It exists purely to *emulate* deployment behavior; a real platform (e.g. Socaity) injects its own implementation (such as a Redis Streams store) via the `stream_store` constructor argument, without changing any endpoint code.
+- **`StreamProducer`** (`engine/streaming/stream_producer.py`) is the bridge the router hands to the worker: it carries the raw chunk iterator, how to serialize each chunk for the store (ChatCompletion deltas, base64-framed media bytes, or plain tokens), the closing chunks (`finish` + `[DONE]`), and how to aggregate the raw chunks into the full `/status` result.
+
+`APIPod()` wires a `LocalStreamStore` automatically whenever a job queue is in use (serverless-localhost and dedicated-with-queue). In plain FastAPI mode (no queue) there is no stream store: streaming happens directly on the request connection (path 1/2 above).
### Deployment
diff --git a/examples/test_clients_six_schemas.py b/test/test_clients_six_schemas.py
similarity index 100%
rename from examples/test_clients_six_schemas.py
rename to test/test_clients_six_schemas.py
diff --git a/examples/test_service_six_schemas.py b/test/test_service_six_schemas.py
similarity index 100%
rename from examples/test_service_six_schemas.py
rename to test/test_service_six_schemas.py
diff --git a/test/test_streaming.py b/test/test_streaming.py
new file mode 100644
index 0000000..870333f
--- /dev/null
+++ b/test/test_streaming.py
@@ -0,0 +1,188 @@
+"""
+Streaming over the serverless-localhost emulation.
+
+APIPod on ``compute="serverless", provider="localhost"`` mimics a real
+deployment: streaming endpoints are queued, the in-process worker produces their
+chunks into a :class:`StreamStore` (the in-memory ``LocalStreamStore`` by
+default), and the client consumes them from ``GET /stream/{job_id}``. A client
+that polls ``GET /status/{job_id}`` instead receives the full aggregated result
+once the job has finished.
+
+These tests exercise three chunk kinds:
+ 1. plain text tokens (any generator endpoint),
+ 2. binary "video" chunks (bytes, base64-framed over SSE),
+ 3. ChatCompletion deltas (schema endpoint with ``stream=true``).
+
+Run directly: python test/test_streaming.py
+Or with pytest: pytest test/test_streaming.py
+"""
+
+import base64
+import time
+
+from fastapi.testclient import TestClient
+
+from apipod import APIPod, LocalStreamStore
+from apipod.common import schemas
+
+
+# Deterministic "video" payload: a few non-trivial binary frames.
+VIDEO_FRAMES = [bytes([i]) * 2048 for i in range(5)]
+VIDEO_BYTES = b"".join(VIDEO_FRAMES)
+
+TEXT_TOKENS = ["APIPod ", "streams ", "tokens ", "one ", "by ", "one."]
+CHAT_TOKENS = ["Hello", ", ", "world", "!"]
+
+
+def build_client() -> TestClient:
+ """Build a serverless-localhost app with streaming endpoints."""
+ app = APIPod(orchestrator="local", compute="serverless", provider="localhost")
+
+ @app.endpoint("/text")
+ def stream_text():
+ for token in TEXT_TOKENS:
+ time.sleep(0.02)
+ yield token
+
+ @app.endpoint("/video")
+ def stream_video():
+ # "Any generator endpoint": here it yields raw video frame bytes.
+ for frame in VIDEO_FRAMES:
+ time.sleep(0.02)
+ yield frame
+
+ @app.endpoint("/chat")
+ def chat(request: schemas.ChatCompletionRequest):
+ if request.stream:
+ def token_gen():
+ for token in CHAT_TOKENS:
+ time.sleep(0.02)
+ yield token
+ return token_gen()
+ return "".join(CHAT_TOKENS)
+
+ fastapi_app = app.app
+ fastapi_app.include_router(app)
+ return TestClient(fastapi_app)
+
+
+def _submit(client: TestClient, path: str, json_body=None) -> str:
+ """POST an endpoint, return the stream URL from the JobResult."""
+ resp = client.post(path, json=json_body)
+ assert resp.status_code == 200, resp.text
+ body = resp.json()
+ assert "job_id" in body, body
+ stream_url = body["links"]["stream"]
+ assert stream_url, body
+ return stream_url
+
+
+def _collect_sse_data(client: TestClient, stream_url: str) -> list[str]:
+ """Open the SSE stream and return the payloads of every ``data:`` line."""
+ payloads: list[str] = []
+ with client.stream("GET", stream_url) as resp:
+ assert resp.status_code == 200, resp.text
+ for line in resp.iter_lines():
+ if not line or line.startswith(":"): # keep-alive / blank
+ continue
+ if line.startswith("data: "):
+ payloads.append(line[len("data: "):])
+ return payloads
+
+
+def test_default_stream_store_is_local():
+ """Serverless localhost gets a LocalStreamStore by default; plain FastAPI does not."""
+ serverless = APIPod(orchestrator="local", compute="serverless", provider="localhost")
+ assert isinstance(serverless.stream_store, LocalStreamStore)
+
+ plain = APIPod(orchestrator="local", compute="dedicated", provider="localhost")
+ assert plain.stream_store is None
+
+
+def test_stream_text_chunks():
+ client = build_client()
+ stream_url = _submit(client, "/text")
+
+ with client.stream("GET", stream_url) as resp:
+ assert resp.status_code == 200, resp.text
+ body = "".join(resp.iter_text())
+
+ assert body == "".join(TEXT_TOKENS)
+
+
+def test_stream_video_bytes():
+ client = build_client()
+ stream_url = _submit(client, "/video")
+
+ payloads = _collect_sse_data(client, stream_url)
+ received = b"".join(base64.b64decode(p) for p in payloads)
+
+ assert received == VIDEO_BYTES
+
+
+def test_stream_chat_completion():
+ client = build_client()
+ stream_url = _submit(client, "/chat", {"messages": [{"role": "user", "content": "hi"}], "stream": True})
+
+ payloads = _collect_sse_data(client, stream_url)
+ assert payloads, "expected at least one chat chunk"
+ assert payloads[-1] == "[DONE]"
+
+ import json
+ content = ""
+ for raw in payloads[:-1]:
+ chunk = json.loads(raw)
+ delta = chunk["choices"][0]["delta"].get("content")
+ if delta:
+ content += delta
+
+ assert content == "".join(CHAT_TOKENS)
+
+
+def test_status_returns_full_result_when_not_streaming():
+ """A client that polls /status (instead of /stream) gets the full result."""
+ client = build_client()
+ resp = client.post("/text")
+ assert resp.status_code == 200, resp.text
+ status_url = resp.json()["links"]["status"]
+
+ # Poll tightly until the job reaches a terminal state and exposes its result.
+ full_result = None
+ deadline = time.time() + 5.0
+ while time.time() < deadline:
+ s = client.get(status_url)
+ body = s.json()
+ if body.get("result") is not None:
+ full_result = body["result"]
+ break
+ if body.get("status") in ("completed", "failed", "not_found"):
+ full_result = body.get("result")
+ break
+ assert full_result == "".join(TEXT_TOKENS)
+
+
+def main() -> int:
+ tests = [
+ test_default_stream_store_is_local,
+ test_stream_text_chunks,
+ test_stream_video_bytes,
+ test_stream_chat_completion,
+ test_status_returns_full_result_when_not_streaming,
+ ]
+ failures = 0
+ for test in tests:
+ try:
+ test()
+ print(f" PASS {test.__name__}")
+ except Exception as exc: # noqa: BLE001
+ failures += 1
+ import traceback
+ print(f" FAIL {test.__name__}: {exc.__class__.__name__}: {exc}")
+ traceback.print_exc()
+ print(f"\n{len(tests) - failures}/{len(tests)} passed")
+ return 1 if failures else 0
+
+
+if __name__ == "__main__":
+ import sys
+ sys.exit(main())
From 49da15fc63730920c5535c15b6eaab45c65ecdc4 Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Mon, 15 Jun 2026 06:06:11 +0200
Subject: [PATCH 12/38] Refactor APIPod core: implement unified streaming,
execution, and job metrics
- Implemented backend-neutral streaming with and centralized serialization logic.
- Unified sync/async function execution across FastAPI and RunPod backends in .
- Centralized endpoint planning and streaming detection into .
- Refactored and to remove platform-specific leaks and unify progress reporting.
- Introduced a dedicated class to consolidate job timing (created, queued, started, finished) and simplify lifecycle tracking.
- Simplified to be self-describing, removing the need for external status mapping.
---
.../backend/fastapi/file_handling_mixin.py | 10 +-
apipod/engine/backend/fastapi/router.py | 11 +-
.../engine/backend/fastapi/streaming_mixin.py | 15 +-
apipod/engine/backend/runpod/router.py | 113 +++--------
apipod/engine/base_backend.py | 63 +++++-
apipod/engine/jobs/base_job.py | 82 ++++----
apipod/engine/jobs/job_progress.py | 55 ++++--
apipod/engine/jobs/job_result.py | 186 +++---------------
apipod/engine/queue/job_queue.py | 26 +--
docs/README_TECH.md | 2 +-
test/test_streaming.py | 2 +-
11 files changed, 207 insertions(+), 358 deletions(-)
diff --git a/apipod/engine/backend/fastapi/file_handling_mixin.py b/apipod/engine/backend/fastapi/file_handling_mixin.py
index f782cab..ce34d22 100644
--- a/apipod/engine/backend/fastapi/file_handling_mixin.py
+++ b/apipod/engine/backend/fastapi/file_handling_mixin.py
@@ -6,7 +6,6 @@
from apipod.common.schemas.media_files import FileModel, ImageFileModel, AudioFileModel, VideoFileModel
from apipod.engine.signatures.policies import FastAPISignaturePolicies
from apipod.engine.files.base_file_mixin import _BaseFileHandlingMixin
-from apipod.engine.endpoint_config import EndpointExecutionPlan
from apipod.engine.utils import replace_func_signature
from media_toolkit import MediaList, MediaDict, ImageFile, AudioFile, VideoFile, MediaFile
import functools
@@ -250,14 +249,9 @@ def _inject_dummy_job_progress(self, func: Callable) -> Callable:
Wrap the function to inject a dummy JobProgress instance if the original
function expects one. This is used for non-queued FastAPI endpoints.
"""
- from apipod.engine.jobs.job_progress import JobProgress
-
- sig = inspect.signature(func)
- job_progress_params = [
- p.name for p in sig.parameters.values()
- if p.name == "job_progress" or "JobProgress" in str(p.annotation)
- ]
+ from apipod.engine.jobs.job_progress import JobProgress, job_progress_param_names
+ job_progress_params = job_progress_param_names(func)
if not job_progress_params:
return func
diff --git a/apipod/engine/backend/fastapi/router.py b/apipod/engine/backend/fastapi/router.py
index b9859e9..3dde124 100644
--- a/apipod/engine/backend/fastapi/router.py
+++ b/apipod/engine/backend/fastapi/router.py
@@ -53,14 +53,12 @@ def __init__(
job_queue: Optional custom JobQueue implementation
lifespan: Optional async context manager for custom startup/shutdown logic
args: Additional arguments
- kwargs: May include ``stream_store`` (SSE backend for GET /stream/{job_id}) and
- ``gateway_stream_url_prefix`` for absolute stream URLs in JobResult, plus
- additional keyword arguments for parent classes.
+ kwargs: May include ``stream_store`` (SSE backend for GET /stream/{job_id}),
+ plus additional keyword arguments for parent classes.
"""
# Extract user-provided lifespan (explicit param or kwarg) before parent init
user_lifespan = lifespan or kwargs.pop('lifespan', None)
stream_store = kwargs.pop("stream_store", None)
- gateway_stream_url_prefix = kwargs.pop("gateway_stream_url_prefix", "")
# Initialize parent classes
api_router_params = inspect.signature(APIRouter.__init__).parameters
@@ -99,7 +97,6 @@ def __init__(
self.app: FastAPI = app
self.prefix = prefix
self.stream_store = stream_store
- self.gateway_stream_url_prefix = gateway_stream_url_prefix
# Let the queue produce streaming job output into the stream store so a
# client can consume it via GET /stream/{job_id} (serverless emulation).
@@ -339,7 +336,7 @@ def _modify_result_decorator(self, func: Callable, plan: EndpointExecutionPlan,
"""
@functools.wraps(func)
def sync_wrapper(*args, **kwargs):
- result = func(*args, **kwargs)
+ result = self.run_callable(func, *args, **kwargs)
producer = build_stream_producer(result, plan.schema_binding)
if producer is not None:
@@ -381,7 +378,7 @@ def decorator(func: Callable) -> Callable:
self._job_func_registry[func.__name__] = func
except Exception:
pass
-
+
upload_enabled = self._prepare_func_for_media_file_upload_with_fastapi(queue_decorated, plan.max_upload_file_size_mb)
return fastapi_route_decorator(upload_enabled)
diff --git a/apipod/engine/backend/fastapi/streaming_mixin.py b/apipod/engine/backend/fastapi/streaming_mixin.py
index 24642d8..a4e3f13 100644
--- a/apipod/engine/backend/fastapi/streaming_mixin.py
+++ b/apipod/engine/backend/fastapi/streaming_mixin.py
@@ -2,9 +2,9 @@
FastAPI-specific streaming transport, factored out of the router.
This mixin owns the FastAPI-side mechanics for turning a streaming endpoint
-result into an HTTP response:
+result into an HTTP response (execution itself goes through the shared
+``_BaseBackend.run_callable`` / ``run_callable_async`` runtime):
-- ``_execute_func`` — run a sync or async function without blocking the event loop;
- ``_stream_generator`` — adapt a sync/async generator into an async generator
for :class:`StreamingResponse` (direct non-queued path);
- ``_streaming_response_from_producer`` — build a :class:`StreamingResponse` from
@@ -45,13 +45,6 @@ class _FastAPIStreamingMixin:
# Execution helpers
# ------------------------------------------------------------------
- async def _execute_func(self, func, **kwargs):
- """Execute a function, handling both sync and async callables."""
- if inspect.iscoroutinefunction(func):
- return await func(**kwargs)
- loop = asyncio.get_event_loop()
- return await loop.run_in_executor(None, functools.partial(func, **kwargs))
-
async def _stream_generator(self, result):
"""Adapt a sync or async generator into an async generator for StreamingResponse."""
if inspect.isasyncgen(result):
@@ -105,7 +98,7 @@ def _create_streaming_endpoint_decorator(self, plan: EndpointExecutionPlan):
def decorator(func: Callable) -> Callable:
@functools.wraps(func)
async def streaming_wrapper(*w_args, **w_kwargs):
- result = await self._execute_func(func, *w_args, **w_kwargs)
+ result = await self.run_callable_async(func, *w_args, **w_kwargs)
generator = self._stream_generator(result)
headers = {"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}
@@ -144,7 +137,7 @@ async def stream_job_sse(self, job_id: str, request: Request):
# Allow any not-yet-finished state; read_chunks waits for the producer.
st = ((job_data.get("status") if job_data else "") or "").lower()
- if not stream_open and st not in {"pending", "processing", "streaming"}:
+ if not stream_open and st not in {"queued", "processing", "streaming"}:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Job '{job_id}' is not streaming (status: {st or 'unknown'}).",
diff --git a/apipod/engine/backend/runpod/router.py b/apipod/engine/backend/runpod/router.py
index 33b6392..6ee2fca 100644
--- a/apipod/engine/backend/runpod/router.py
+++ b/apipod/engine/backend/runpod/router.py
@@ -1,4 +1,3 @@
-import asyncio
import functools
import inspect
import traceback
@@ -6,15 +5,9 @@
from typing import Union, Callable, Iterator
from apipod.common import constants
-from apipod.engine.jobs.base_job import JOB_STATUS
-from apipod.engine.jobs.job_progress import JobProgressRunpod, JobProgress
-from apipod.engine.jobs.job_result import (
- JobResultFactory,
- JobResult,
- JobMetrics,
- _compute_duration_s,
- _job_status_to_public,
-)
+from apipod.engine.jobs.base_job import BaseJob, JOB_STATUS
+from apipod.engine.jobs.job_progress import JobProgressRunpod, job_progress_param_names
+from apipod.engine.jobs.job_result import JobResultFactory
from apipod.engine.base_backend import _BaseBackend
from apipod.engine.endpoint_config import build_plan, EndpointExecutionPlan
from apipod.engine.files.base_file_mixin import _BaseFileHandlingMixin
@@ -71,7 +64,7 @@ def wrapper(*w_args, **w_kwargs):
try:
if plan.is_schema_endpoint:
return self._handle_schema_request(func, plan.schema_binding, w_args, w_kwargs)
- result = self._execute_sync_or_async(func, w_args, w_kwargs)
+ result = self.run_callable(func, *w_args, **w_kwargs)
# Plain generator endpoints stream too: turn the generator into a
# RunPod-native stream of JSON-safe chunks (RunPod aggregates it).
native_stream = self._as_native_stream(result)
@@ -91,7 +84,7 @@ def _handle_schema_request(self, func: Callable, binding: SchemaBinding, args: t
registered response model (``wrap_schema_response``).
"""
request = prepare_schema_call(binding, kwargs)
- result = self._execute_sync_or_async(func, args, kwargs)
+ result = self.run_callable(func, *args, **kwargs)
if getattr(request, "stream", False):
native_stream = self._as_native_stream(result, binding)
@@ -122,19 +115,6 @@ def _as_native_stream(self, result, binding: Union[SchemaBinding, None] = None)
return None
- def _execute_sync_or_async(self, func, args, kwargs):
- if inspect.iscoroutinefunction(func):
- return self._run_in_loop(func(*args, **kwargs))
- return func(*args, **kwargs)
-
- def _run_in_loop(self, coro):
- try:
- loop = asyncio.get_event_loop()
- except RuntimeError:
- loop = asyncio.new_event_loop()
- asyncio.set_event_loop(loop)
- return loop.run_until_complete(coro)
-
def get(self, path: str = None, *args, **kwargs):
return self.endpoint(path=path, *args, **kwargs)
@@ -153,11 +133,7 @@ def _add_job_progress_to_kwargs(self, func, job, kwargs):
Returns:
Updated kwargs with job_progress added
"""
- job_progress_params = []
- for param in inspect.signature(func).parameters.values():
- if param.annotation in (JobProgress, JobProgressRunpod) or param.name == "job_progress":
- job_progress_params.append(param.name)
-
+ job_progress_params = job_progress_param_names(func)
if job_progress_params:
jp = JobProgressRunpod(job)
for job_progress_param in job_progress_params:
@@ -196,82 +172,37 @@ def _router(self, path, job, **kwargs):
if missing_args:
raise Exception(f"Arguments {missing_args} are missing")
- start_time = datetime.now(timezone.utc)
- result = JobResult(job_id=job["id"], endpoint=path)
+ # Create a BaseJob record so the result can be assembled by the same
+ # from_base_job path used by the local queue worker.
+ job_record = BaseJob(id=job["id"])
+ job_record.metrics.started_at = datetime.now(timezone.utc)
try:
- # Execute the function (Sync or Async Handling)
- res = self._execute_route_function(route_function, kwargs)
+ res = self.run_callable(route_function, **kwargs)
- # Check if result is a generator (streaming response)
+ # Streaming response: hand the generator straight to RunPod, which
+ # aggregates it (no JobResult envelope is produced for streams).
if inspect.isgenerator(res) or inspect.isasyncgen(res):
- # For streaming, return the generator directly
- # RunPod will handle the streaming
return res
- # Convert result to JSON if it's a MediaFile / MediaList / Pydantic Model
- res = JobResultFactory._serialize_result(res)
-
- result.result = res
- result.status = _job_status_to_public(JOB_STATUS.FINISHED)
+ job_record.result = res
+ job_record.status = JOB_STATUS.FINISHED
except Exception as e:
- result.error = str(e)
- result.status = _job_status_to_public(JOB_STATUS.FAILED)
+ job_record.error = str(e)
+ job_record.status = JOB_STATUS.FAILED
print(f"Job {job['id']} failed: {str(e)}")
traceback.print_exc()
+ # Adopt the progress handle the function reported through, so its final
+ # progress/message flow into the JobResult.
for arg in kwargs.values():
if isinstance(arg, JobProgressRunpod):
- result.progress = float(arg._progress)
- result.message = arg._message
+ job_record.job_progress = arg
break
- inference_time_s = _compute_duration_s(start_time, datetime.now(timezone.utc))
- if inference_time_s is not None:
- result.metrics = JobMetrics(inference_time_s=inference_time_s)
-
- return result.model_dump_json(exclude_none=True)
+ job_record.metrics.finished_at = datetime.now(timezone.utc)
- def _execute_route_function(self, route_function, kwargs):
- """
- Execute a route function, handling both sync and async functions.
-
- Args:
- route_function: The function to execute
- kwargs: Keyword arguments to pass to the function
-
- Returns:
- The result of the function execution
- """
- if not inspect.iscoroutinefunction(route_function):
- # Synchronous function - simple execution
- return route_function(**kwargs)
-
- # Async function - need event loop handling
- try:
- loop = asyncio.get_event_loop()
- if loop.is_running():
- # Already in async context - shouldn't happen in RunPod handler
- # But if it does, we need to handle it differently
- import concurrent.futures
- with concurrent.futures.ThreadPoolExecutor() as executor:
- future = executor.submit(
- asyncio.run,
- route_function(**kwargs)
- )
- return future.result()
- else:
- # Loop exists but not running - use it
- return loop.run_until_complete(route_function(**kwargs))
- except RuntimeError:
- # No event loop exists - create one
- loop = asyncio.new_event_loop()
- asyncio.set_event_loop(loop)
- try:
- return loop.run_until_complete(route_function(**kwargs))
- finally:
- loop.close()
- asyncio.set_event_loop(None)
+ return JobResultFactory.from_base_job(job_record).model_dump_json(exclude_none=True)
def handler(self, job):
"""
diff --git a/apipod/engine/base_backend.py b/apipod/engine/base_backend.py
index 64cd5d2..e1316f2 100644
--- a/apipod/engine/base_backend.py
+++ b/apipod/engine/base_backend.py
@@ -1,5 +1,9 @@
+import asyncio
+import concurrent.futures
+import functools
+import inspect
from abc import abstractmethod
-from typing import Union
+from typing import Any, Callable, Union
import importlib.metadata
from apipod.common import constants
from apipod.engine.compatibility.HealthCheck import HealthCheck
@@ -23,6 +27,61 @@ def __init__(
self._health_check = HealthCheck()
self.version = importlib.metadata.version("apipod")
+ # ------------------------------------------------------------------
+ # Execution runtime
+ #
+ # Executing a user-registered endpoint while transparently resolving
+ # sync/async is the one operation every backend shares. There are exactly
+ # two authoritative entry points so behaviour can never drift between the
+ # worker thread, the RunPod handler and the direct FastAPI path.
+ # ------------------------------------------------------------------
+ def run_callable(self, func: Callable, *args: Any, **kwargs: Any) -> Any:
+ """Execute *func* from a synchronous context, resolving coroutines.
+
+ Used by the two real sync execution sites: the local queue worker thread
+ and the RunPod handler. Sync functions run directly; coroutine functions
+ are driven to completion on an event loop (a fresh one is created and torn
+ down when no usable loop exists, or when the current loop is already
+ running the call is offloaded to a worker thread).
+ """
+ if not inspect.iscoroutinefunction(func):
+ return func(*args, **kwargs)
+
+ coro = func(*args, **kwargs)
+ try:
+ loop = asyncio.get_event_loop()
+ except RuntimeError:
+ loop = None
+
+ if loop is not None and not loop.is_running():
+ return loop.run_until_complete(coro)
+
+ if loop is not None and loop.is_running():
+ # Already inside a running loop (rare in sync handlers): run the
+ # coroutine to completion on a separate thread to avoid re-entrancy.
+ with concurrent.futures.ThreadPoolExecutor() as executor:
+ return executor.submit(asyncio.run, coro).result()
+
+ new_loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(new_loop)
+ try:
+ return new_loop.run_until_complete(coro)
+ finally:
+ new_loop.close()
+ asyncio.set_event_loop(None)
+
+ async def run_callable_async(self, func: Callable, *args: Any, **kwargs: Any) -> Any:
+ """Execute *func* from an async context without blocking the event loop.
+
+ Used by the direct (non-queued) FastAPI response path. Coroutine
+ functions are awaited; sync functions are offloaded to the default
+ thread-pool executor.
+ """
+ if inspect.iscoroutinefunction(func):
+ return await func(*args, **kwargs)
+ loop = asyncio.get_event_loop()
+ return await loop.run_in_executor(None, functools.partial(func, *args, **kwargs))
+
@property
def status(self) -> constants.SERVER_HEALTH:
return self._health_check.status
@@ -60,7 +119,7 @@ def endpoint(self, path: str = None, *args, **kwargs):
"""
Add a route to the app.
Can be a task route (async job) or standard route depending on configuration.
-
+
:param path:
In case of fastapi will be resolved as url in form http://{host:port}/{prefix}/{path}
In case of runpod will be resolved as url in form http://{host:port}?route={path}
diff --git a/apipod/engine/jobs/base_job.py b/apipod/engine/jobs/base_job.py
index 655f861..411429a 100644
--- a/apipod/engine/jobs/base_job.py
+++ b/apipod/engine/jobs/base_job.py
@@ -7,42 +7,46 @@
class JOB_STATUS(Enum):
- QUEUED = "Queued"
- PROCESSING = "Processing"
- STREAMING = "Streaming"
- FINISHED = "Finished"
- FAILED = "Failed"
- TIMEOUT = "Timeout"
+ """Lifecycle states of a job. The value is the public API status string."""
+ QUEUED = "queued"
+ PROCESSING = "processing"
+ STREAMING = "streaming"
+ FINISHED = "finished"
+ FAILED = "failed"
+ TIMEOUT = "timeout"
-class PROVIDERS(Enum):
- RUNPOD = "runpod"
- OPENAI = "openai"
- REPLICATE = "replicate"
+class JobMetrics:
+ """Storage for all job timing and performance data."""
+ def __init__(self):
+ self.created_at: datetime = datetime.now(timezone.utc)
+ self.queued_at: Optional[datetime] = None
+ self.started_at: Optional[datetime] = None
+ self.finished_at: Optional[datetime] = None
+ self.time_out_at: Optional[datetime] = None
-class BaseJob:
- """Essential job record shared by all job types (local thread, remote service, etc.).
-
- Subclasses add domain-specific fields:
+ @property
+ def execution_time_s(self) -> Optional[float]:
+ if self.started_at and self.finished_at:
+ delta = (self.finished_at - self.started_at).total_seconds()
+ return round(delta, 2) if delta >= 0 else 0.0
+ return None
- * :class:`LocalJob` — thread-pool execution (``job_function``, ``job_progress``).
- * Platform-specific subclasses (e.g. gateway ``ServiceJob``) — remote/Redis
- (``service_id``, ``api_url``, ``endpoint``, ``input_data``).
- :class:`~apipod.engine.jobs.job_result.JobResultFactory.from_base_job` maps any
- ``BaseJob`` subclass to the public :class:`~apipod.engine.jobs.job_result.JobResult`.
+class BaseJob:
+ """
+ Essential job record shared by all job types (local thread, remote service, etc.).
+ Subclass to add domain-specific fields like service_id, endpoint, etc.
"""
- def __init__(self, id=None, status="pending", created_at=None, updated_at=None):
+ def __init__(self, id: Optional[str] = None):
self.id: str = id or str(uuid4())
- self.status = status
+ self.status = JOB_STATUS.QUEUED
self.result: Any = None
self.error: Optional[str] = None
- self.created_at = created_at or datetime.now(timezone.utc)
- self.updated_at = updated_at or datetime.now(timezone.utc)
- self.progress: Optional[float] = None
- self.message: Optional[str] = None
+ self.job_progress = JobProgress()
+ self.metrics = JobMetrics()
class LocalJob(BaseJob):
@@ -51,31 +55,13 @@ class LocalJob(BaseJob):
"""
def __init__(self, job_function: callable, job_params: Optional[dict] = None, timeout_seconds: int = 3600):
- super().__init__(status=JOB_STATUS.QUEUED)
+ super().__init__()
self.job_function = job_function
self.job_params: dict = job_params or {}
- self.job_progress = JobProgress()
-
- self.queued_at: Optional[datetime] = None
- self.execution_started_at: Optional[datetime] = None
- self.execution_finished_at: Optional[datetime] = None
- self.time_out_at = self.created_at + timedelta(seconds=timeout_seconds)
+ self.metrics.time_out_at = self.metrics.created_at + timedelta(seconds=timeout_seconds)
@property
def is_timed_out(self) -> bool:
- return datetime.now(timezone.utc) > self.time_out_at
-
- @property
- def execution_duration_ms(self) -> int:
- if not self.execution_started_at:
- return 0
- end_time = self.execution_finished_at or datetime.now(timezone.utc)
- return int((end_time - self.execution_started_at).total_seconds() * 1000)
-
- @property
- def delay_time_ms(self) -> int:
- if not self.queued_at:
- return int((datetime.now(timezone.utc) - self.created_at).total_seconds() * 1000)
- if not self.execution_started_at:
- return int((datetime.now(timezone.utc) - self.queued_at).total_seconds() * 1000)
- return int((self.execution_started_at - self.created_at).total_seconds() * 1000)
+ if not self.metrics.time_out_at:
+ return False
+ return datetime.now(timezone.utc) > self.metrics.time_out_at
diff --git a/apipod/engine/jobs/job_progress.py b/apipod/engine/jobs/job_progress.py
index b9b6ed5..8e4d062 100644
--- a/apipod/engine/jobs/job_progress.py
+++ b/apipod/engine/jobs/job_progress.py
@@ -1,29 +1,54 @@
+import inspect
import logging
+from typing import Callable, List
+
logger = logging.getLogger(__name__)
+def job_progress_param_names(func: Callable) -> List[str]:
+ """Names of the parameters of *func* that should receive a :class:`JobProgress`.
+
+ A parameter qualifies when it is literally named ``job_progress`` or when its
+ annotation refers to a ``JobProgress`` type. This single detection is shared
+ by every injection site (queue worker, RunPod handler, direct FastAPI path)
+ so they can never disagree about what counts as a progress parameter.
+ """
+ try:
+ params = inspect.signature(func).parameters.values()
+ except (TypeError, ValueError):
+ return []
+ return [
+ p.name for p in params
+ if p.name == "job_progress" or "JobProgress" in str(p.annotation)
+ ]
+
+
class JobProgress:
- def __init__(self, progress: float = 0, message: str = None):
- """
- Used to display _progress of a job while executing.
- :param progress: value between 0 and 1.0
- :param message: message to deliver to client.
- """
- self._progress = progress
- self._message = message
+ """Live progress handle for a running job.
+
+ Holds the single source of truth for a job's ``progress`` and ``message``:
+ the endpoint function receives this object (via a ``job_progress`` parameter)
+ and reports updates through :meth:`set_status`, while the job record exposes
+ the same object so the public ``JobResult`` can read the current values.
+ """
+
+ def __init__(self, progress: float = 0.0, message: str = None):
+ """:param progress: value between 0 and 1.0. :param message: message to deliver to the client."""
+ self.progress = progress
+ self.message = message
logger.setLevel(logging.INFO)
-
+
def set_status(self, progress: float = None, message: str = None):
if progress is not None:
- self._progress = progress
+ self.progress = progress
if message is not None:
- self._message = message
- logger.info(f"Progress: {self._progress} Message: {self._message}")
+ self.message = message
+ logger.info(f"Progress: {self.progress} Message: {self.message}")
return self
-
+
class JobProgressRunpod(JobProgress):
- def __init__(self, runpod_job, progress: float = 0, message: str = None):
+ def __init__(self, runpod_job, progress: float = 0.0, message: str = None):
super().__init__(progress=progress, message=message)
self.runpod_job = runpod_job
@@ -34,7 +59,7 @@ def set_status(self, progress: float = None, message: str = None):
import runpod
runpod.serverless.progress_update(
self.runpod_job,
- f"Progress: {int(self._progress)} Message: {self._message}"
+ f"Progress: {int(self.progress)} Message: {self.message}"
)
except Exception as e:
print(f"Problem in progress update: {e}")
diff --git a/apipod/engine/jobs/job_result.py b/apipod/engine/jobs/job_result.py
index 8523f90..1c8d76d 100644
--- a/apipod/engine/jobs/job_result.py
+++ b/apipod/engine/jobs/job_result.py
@@ -1,10 +1,10 @@
import gzip
+from datetime import datetime
from io import BytesIO
from typing import Any, List, Optional, Union
from pydantic import BaseModel
-from apipod.common.settings import DEFAULT_DATE_TIME_FORMAT
from apipod.engine.jobs.base_job import JOB_STATUS, BaseJob
from apipod.engine.signatures.upload import is_param_media_toolkit_file
from media_toolkit import IMediaContainer
@@ -12,115 +12,36 @@
from apipod.common.schemas.media_files import FileModel
-def _job_status_to_public(status: Any) -> Optional[str]:
- """Map internal JOB_STATUS (or legacy string) to public API strings (gateway-aligned)."""
+def _public_status(status: Any) -> Optional[str]:
+ """Public status string: a JOB_STATUS carries its own value; pass strings through."""
if status is None:
return None
- if isinstance(status, JOB_STATUS):
- return {
- JOB_STATUS.QUEUED: "pending",
- JOB_STATUS.PROCESSING: "processing",
- JOB_STATUS.STREAMING: "streaming",
- JOB_STATUS.FINISHED: "completed",
- JOB_STATUS.FAILED: "failed",
- JOB_STATUS.TIMEOUT: "failed",
- }.get(status, status.value.lower())
- if isinstance(status, str):
- lowered = status.lower()
- legacy = {
- "queued": "pending",
- "pending": "pending",
- "processing": "processing",
- "streaming": "streaming",
- "finished": "completed",
- "completed": "completed",
- "failed": "failed",
- "timeout": "failed",
- "rejected": "failed",
- }
- return legacy.get(lowered, lowered)
- return str(status)
-
-
-def _format_date(date: Any) -> Optional[str]:
- """Format a datetime or ISO string for the public API."""
- if date is None:
- return None
- if isinstance(date, str):
- return date
- try:
- return date.strftime(DEFAULT_DATE_TIME_FORMAT)
- except Exception:
- return str(date)
-
-
-def _parse_iso(value: Any):
- """Best-effort parse an ISO 8601 string or datetime into a datetime."""
- if value is None:
- return None
- if hasattr(value, "timestamp"):
- return value
- try:
- from datetime import datetime, timezone
- dt = datetime.fromisoformat(str(value))
- if dt.tzinfo is None:
- dt = dt.replace(tzinfo=timezone.utc)
- return dt
- except (ValueError, TypeError):
- return None
-
-
-def _compute_duration_s(start: Any, end: Any) -> Optional[float]:
- """Compute seconds between two timestamps, returning None if either is missing."""
- s, e = _parse_iso(start), _parse_iso(end)
- if s is None or e is None:
- return None
- delta = (e - s).total_seconds()
- return round(delta, 2) if delta >= 0 else None
-
-
-def _opt_float(value: Any) -> Optional[float]:
- """Coerce to positive float, else None."""
- if value is None:
- return None
- try:
- f = float(value)
- return round(f, 3) if f > 0 else None
- except (ValueError, TypeError):
- return None
+ return status.value if isinstance(status, JOB_STATUS) else str(status)
class JobLinks(BaseModel):
"""Hypermedia links for job status polling, cancellation, and streaming."""
-
status: Optional[str] = None
cancel: Optional[str] = None
stream: Optional[str] = None
class JobMetrics(BaseModel):
- """Performance metrics populated as a job progresses through the platform.
-
- Segments (chronological):
- upload_time_s – file upload duration (gateway)
- platform_queue_time_s – our validation + dispatch + Celery routing
- provider_queue_time_s – provider-side GPU / resource wait
- inference_time_s – actual model execution
- execution_time_s – orchestrator end-to-end (queue + inference, excludes upload)
- """
-
+ """Execution metrics APIPod measures for a job."""
+ created_at: Optional[datetime] = None
+ queued_at: Optional[datetime] = None
+ started_at: Optional[datetime] = None
+ finished_at: Optional[datetime] = None
execution_time_s: Optional[float] = None
- inference_time_s: Optional[float] = None
- platform_queue_time_s: Optional[float] = None
- provider_queue_time_s: Optional[float] = None
- upload_time_s: Optional[float] = None
class JobResult(BaseModel):
"""Public job snapshot returned by GET /status and job submissions.
Unified response: same shape whether the client just submitted a job
- (``status="pending"``) or is polling for completion.
+ (``status="queued"``) or is polling for completion. The ``status`` values
+ mirror :class:`~apipod.engine.jobs.base_job.JOB_STATUS`
+ (``queued``/``processing``/``streaming``/``finished``/``failed``/``timeout``).
Null fields are excluded from the serialized response so the client
only receives relevant information for the current job state.
@@ -133,9 +54,6 @@ class JobResult(BaseModel):
progress: Optional[float] = None
message: Optional[str] = None
- service: Optional[str] = None
- endpoint: Optional[str] = None
-
metrics: Optional[JobMetrics] = None
links: Optional[JobLinks] = None
@@ -176,75 +94,29 @@ def _serialize_result(data: Any) -> Union[FileModel, List[FileModel], list, str,
@staticmethod
def from_base_job(job: BaseJob) -> JobResult:
- """Map any :class:`BaseJob` subclass to the public :class:`JobResult`.
-
- Works for :class:`~apipod.engine.jobs.base_job.LocalJob` (thread queue)
- and any platform ``BaseJob`` subclass (e.g. gateway ``ServiceJob``).
- """
- status = _job_status_to_public(job.status)
- result = JobResultFactory._serialize_result(job.result)
-
- progress = job.progress
- message = job.message
-
- job_progress = getattr(job, "job_progress", None)
- if job_progress is not None:
- try:
- progress = float(job_progress._progress)
- message = job_progress._message
- except Exception:
- pass
-
- service = getattr(job, "service_id", None)
- endpoint = getattr(job, "endpoint", None)
-
- metrics = JobResultFactory._build_metrics(job)
-
- links = JobLinks(
- status=f"/status/{job.id}",
- cancel=f"/cancel/{job.id}",
- stream=f"/stream/{job.id}",
+ """Map a :class:`BaseJob` to the public :class:`JobResult`."""
+ m = job.metrics
+ metrics = JobMetrics(
+ created_at=m.created_at,
+ queued_at=m.queued_at,
+ started_at=m.started_at,
+ finished_at=m.finished_at,
+ execution_time_s=m.execution_time_s,
)
return JobResult(
job_id=job.id,
- status=status,
+ status=_public_status(job.status),
+ result=JobResultFactory._serialize_result(job.result),
error=job.error,
- result=result,
- progress=progress,
- message=message,
- service=service,
- endpoint=endpoint,
+ progress=job.job_progress.progress,
+ message=job.job_progress.message,
metrics=metrics,
- links=links,
- )
-
- @staticmethod
- def _build_metrics(job: BaseJob) -> Optional[JobMetrics]:
- """Derive timing metrics from orchestrator-provided values or timestamps."""
- execution_time_s = _opt_float(getattr(job, "execution_time_s", None)) or _compute_duration_s(
- getattr(job, "created_at", None),
- getattr(job, "completed_at", None) or getattr(job, "failed_at", None),
- )
- upload_time_s = _compute_duration_s(
- getattr(job, "upload_started_at", None),
- getattr(job, "upload_finished_at", None),
- )
- inference_time_s = _opt_float(getattr(job, "inference_time_s", None))
- platform_queue_time_s = _opt_float(getattr(job, "platform_queue_time_s", None))
- provider_queue_time_s = _opt_float(getattr(job, "provider_queue_time_s", None))
-
- values = (execution_time_s, upload_time_s, inference_time_s,
- platform_queue_time_s, provider_queue_time_s)
- if all(v is None for v in values):
- return None
-
- return JobMetrics(
- execution_time_s=execution_time_s,
- inference_time_s=inference_time_s,
- platform_queue_time_s=platform_queue_time_s,
- provider_queue_time_s=provider_queue_time_s,
- upload_time_s=upload_time_s,
+ links=JobLinks(
+ status=f"/status/{job.id}",
+ cancel=f"/cancel/{job.id}",
+ stream=f"/stream/{job.id}",
+ ),
)
@staticmethod
diff --git a/apipod/engine/queue/job_queue.py b/apipod/engine/queue/job_queue.py
index 1628d49..ee9dd92 100644
--- a/apipod/engine/queue/job_queue.py
+++ b/apipod/engine/queue/job_queue.py
@@ -6,10 +6,9 @@
from apipod.engine.queue.job_store import JobStore
from apipod.engine.jobs.base_job import BaseJob, LocalJob, JOB_STATUS
-from apipod.engine.jobs.job_result import _job_status_to_public
+from apipod.engine.jobs.job_progress import job_progress_param_names
from apipod.engine.queue.job_queue_interface import JobQueueInterface
from apipod.engine.streaming.stream_producer import StreamProducer
-import inspect
T = TypeVar('T', bound=BaseJob)
@@ -51,7 +50,7 @@ def get_job_status(self, job_id: str) -> Optional[dict]:
job = self.job_store.get_job(job_id)
if job is None:
return None
- return {"id": job_id, "status": _job_status_to_public(job.status)}
+ return {"id": job_id, "status": getattr(job.status, "value", job.status)}
def set_queue_size(self, job_function: callable, queue_size: int = 500) -> None:
self.queue_sizes[job_function.__name__] = queue_size
@@ -86,7 +85,7 @@ def _add_job(self, job_function: callable, job_params: Optional[dict] = None) ->
return job
job.status = JOB_STATUS.QUEUED
- job.queued_at = datetime.now(timezone.utc)
+ job.metrics.queued_at = datetime.now(timezone.utc)
self.job_store.add_to_queue(job)
if not self.worker_thread.is_alive():
@@ -105,7 +104,7 @@ def _create_job(self, job_function: callable, job_params: Optional[dict] = None)
def _process_job(self, job: T) -> None:
try:
- job.execution_started_at = datetime.now(timezone.utc)
+ job.metrics.started_at = datetime.now(timezone.utc)
job.status = JOB_STATUS.PROCESSING
self._inject_job_progress(job)
@@ -164,7 +163,7 @@ def _run_stream(self, job: T, producer: StreamProducer):
def _complete_job(self, job: T, final_state: JOB_STATUS) -> T:
self.job_store.complete_job(job.id)
# setting status here, because if this is done earlier, race conditions in get_job are the problem
- job.execution_finished_at = datetime.now(timezone.utc)
+ job.metrics.finished_at = datetime.now(timezone.utc)
job.status = final_state
return job
@@ -181,15 +180,8 @@ def cancel_job(self, job_id: str) -> None:
# self._complete_job(job_id)
def _inject_job_progress(self, job: T) -> T:
- sig = inspect.signature(job.job_function)
-
- job_progress_params = [
- p for p in sig.parameters.values()
- if p.name == "job_progress" or "JobProgress" in str(p.annotation)
- ]
- for job_progress_param in job_progress_params:
- job.job_params[job_progress_param.name] = job.job_progress
-
+ for param_name in job_progress_param_names(job.job_function):
+ job.job_params[param_name] = job.job_progress
return job
def _process_jobs_in_background(self) -> None:
@@ -244,9 +236,9 @@ def _clean_up_orphan_jobs(self) -> None:
return
for job in self.job_store.completed_jobs:
- if job.execution_finished_at is None:
+ if job.metrics.finished_at is None:
self._remove_job(job)
- elif (datetime.now(timezone.utc) - job.execution_finished_at).total_seconds() > self._delete_orphan_jobs_after_seconds:
+ elif (datetime.now(timezone.utc) - job.metrics.finished_at).total_seconds() > self._delete_orphan_jobs_after_seconds:
self._remove_job(job)
def _start_queued_jobs(self) -> None:
diff --git a/docs/README_TECH.md b/docs/README_TECH.md
index b8f1240..85ba046 100644
--- a/docs/README_TECH.md
+++ b/docs/README_TECH.md
@@ -130,6 +130,6 @@ The **stream store** is the pluggable backend that buffers a job's chunks betwee
## Request lifecycle, end to end
-A client calls `POST /tts` with a JSON body. FastAPI validates it against the rewritten signature and calls the outermost wrapper. The file-handling layer converts any media inputs into media-toolkit objects. If the endpoint is queued, the queue mixin stores the job and returns `{job_id, status: "pending", links}` — the worker thread later executes the real function, feeding `JobProgress` updates into the store. The client (typically fastSDK) polls `/status/{job_id}` until `completed` and receives the result, with any returned `AudioFile` serialized as a `FileModel`. Without a queue the same conversion happens inline and the response returns directly. On RunPod, the identical user function is reached through `handler → _router(path) → file handling → execute`, proving the core principle: the function is written once, the backend decides how it runs.
+A client calls `POST /tts` with a JSON body. FastAPI validates it against the rewritten signature and calls the outermost wrapper. The file-handling layer converts any media inputs into media-toolkit objects. If the endpoint is queued, the queue mixin stores the job and returns `{job_id, status: "queued", links}` — the worker thread later executes the real function, feeding `JobProgress` updates into the store. The client (typically fastSDK) polls `/status/{job_id}` until `finished` and receives the result, with any returned `AudioFile` serialized as a `FileModel`. Without a queue the same conversion happens inline and the response returns directly. On RunPod, the identical user function is reached through `handler → _router(path) → file handling → execute`, proving the core principle: the function is written once, the backend decides how it runs.
diff --git a/test/test_streaming.py b/test/test_streaming.py
index 870333f..f52d059 100644
--- a/test/test_streaming.py
+++ b/test/test_streaming.py
@@ -155,7 +155,7 @@ def test_status_returns_full_result_when_not_streaming():
if body.get("result") is not None:
full_result = body["result"]
break
- if body.get("status") in ("completed", "failed", "not_found"):
+ if body.get("status") in ("finished", "failed", "timeout", "not_found"):
full_result = body.get("result")
break
assert full_result == "".join(TEXT_TOKENS)
From 003725d662c1e8a5720e947bf651c86d30edd63e Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Mon, 15 Jun 2026 06:24:41 +0200
Subject: [PATCH 13/38] housekeeping regarding function analysis methods.
Moving them into the right files
---
.../backend/fastapi/file_handling_mixin.py | 4 +-
apipod/engine/backend/runpod/router.py | 3 +-
apipod/engine/backend/schema_resolve.py | 6 +--
apipod/engine/endpoint_config.py | 33 ++-----------
apipod/engine/jobs/job_progress.py | 20 --------
apipod/engine/queue/job_queue.py | 2 +-
apipod/engine/signatures/analysis.py | 47 +++++++++++++++++++
7 files changed, 57 insertions(+), 58 deletions(-)
create mode 100644 apipod/engine/signatures/analysis.py
diff --git a/apipod/engine/backend/fastapi/file_handling_mixin.py b/apipod/engine/backend/fastapi/file_handling_mixin.py
index ce34d22..4913d98 100644
--- a/apipod/engine/backend/fastapi/file_handling_mixin.py
+++ b/apipod/engine/backend/fastapi/file_handling_mixin.py
@@ -7,6 +7,8 @@
from apipod.engine.signatures.policies import FastAPISignaturePolicies
from apipod.engine.files.base_file_mixin import _BaseFileHandlingMixin
from apipod.engine.utils import replace_func_signature
+from apipod.engine.signatures.analysis import job_progress_param_names
+from apipod.engine.jobs.job_progress import JobProgress
from media_toolkit import MediaList, MediaDict, ImageFile, AudioFile, VideoFile, MediaFile
import functools
@@ -249,8 +251,6 @@ def _inject_dummy_job_progress(self, func: Callable) -> Callable:
Wrap the function to inject a dummy JobProgress instance if the original
function expects one. This is used for non-queued FastAPI endpoints.
"""
- from apipod.engine.jobs.job_progress import JobProgress, job_progress_param_names
-
job_progress_params = job_progress_param_names(func)
if not job_progress_params:
return func
diff --git a/apipod/engine/backend/runpod/router.py b/apipod/engine/backend/runpod/router.py
index 6ee2fca..07f35ee 100644
--- a/apipod/engine/backend/runpod/router.py
+++ b/apipod/engine/backend/runpod/router.py
@@ -6,10 +6,11 @@
from apipod.common import constants
from apipod.engine.jobs.base_job import BaseJob, JOB_STATUS
-from apipod.engine.jobs.job_progress import JobProgressRunpod, job_progress_param_names
+from apipod.engine.jobs.job_progress import JobProgressRunpod
from apipod.engine.jobs.job_result import JobResultFactory
from apipod.engine.base_backend import _BaseBackend
from apipod.engine.endpoint_config import build_plan, EndpointExecutionPlan
+from apipod.engine.signatures.analysis import job_progress_param_names
from apipod.engine.files.base_file_mixin import _BaseFileHandlingMixin
from apipod.engine.backend.schema_resolve import (
SchemaBinding,
diff --git a/apipod/engine/backend/schema_resolve.py b/apipod/engine/backend/schema_resolve.py
index 5f7d4e6..3ec988c 100644
--- a/apipod/engine/backend/schema_resolve.py
+++ b/apipod/engine/backend/schema_resolve.py
@@ -31,6 +31,7 @@
from apipod.common.schemas import *
from apipod.engine.files.base_file_mixin import parse_schema_media_fields
+from apipod.engine.signatures.analysis import is_injected_progress_param
@dataclass(frozen=True)
@@ -126,10 +127,7 @@ def get_schema_binding(func: Callable) -> Optional[SchemaBinding]:
def _is_injected_param(param: inspect.Parameter) -> bool:
"""True for parameters the framework supplies (job progress, self/cls)."""
- return (
- param.name in ("self", "cls", "job_progress")
- or "JobProgress" in str(param.annotation)
- )
+ return param.name in ("self", "cls") or is_injected_progress_param(param)
def _validate_schema_endpoint_signature(func: Callable, binding: SchemaBinding) -> None:
diff --git a/apipod/engine/endpoint_config.py b/apipod/engine/endpoint_config.py
index db10bac..1eca91c 100644
--- a/apipod/engine/endpoint_config.py
+++ b/apipod/engine/endpoint_config.py
@@ -1,19 +1,12 @@
"""
Endpoint planning: backend-neutral analysis of an endpoint function.
-
-``is_streaming_endpoint`` inspects a function's definition to decide whether
-it streams its output. ``build_plan`` combines that with schema detection and
-optional backend-specific parameters (queue, upload limits, …) into an
-immutable :class:`EndpointExecutionPlan` — the shared contract used by every
-backend (FastAPI, RunPod).
"""
-import inspect
-from collections.abc import AsyncIterator, Iterator
from dataclasses import dataclass, field
-from typing import Any, Callable, get_origin, get_type_hints
+from typing import Any, Callable, Optional
from apipod.engine.backend.schema_resolve import SchemaBinding, get_schema_binding
+from apipod.engine.signatures.analysis import is_streaming_endpoint
@dataclass(frozen=True)
@@ -45,26 +38,6 @@ def active_methods(self) -> list[str]:
return ["POST"] if self.methods is None else self.methods
-def is_streaming_endpoint(func: Callable) -> bool:
- """Backend-neutral: True if *func* is a generator or annotated as an Iterator.
-
- A generator function (``yield``) or async generator function (``async yield``)
- is always considered streaming. A regular function whose return annotation
- is ``Iterator[...]`` / ``AsyncIterator[...]`` or any subclass is also detected.
- """
- target = inspect.unwrap(func)
- if inspect.isgeneratorfunction(target) or inspect.isasyncgenfunction(target):
- return True
- try:
- return_type = get_type_hints(target).get("return")
- except Exception:
- return False
- if return_type is None:
- return False
- origin = get_origin(return_type) or return_type
- return inspect.isclass(origin) and issubclass(origin, (Iterator, AsyncIterator))
-
-
def build_plan(
func: Callable,
path: str,
@@ -92,5 +65,5 @@ def build_plan(
route_args=route_args,
route_kwargs=route_kwargs if route_kwargs is not None else {},
schema_binding=schema_binding,
- is_streaming=schema_binding is None and is_streaming_endpoint(func),
+ is_streaming=schema_binding is None and is_streaming_endpoint(func)
)
diff --git a/apipod/engine/jobs/job_progress.py b/apipod/engine/jobs/job_progress.py
index 8e4d062..9817909 100644
--- a/apipod/engine/jobs/job_progress.py
+++ b/apipod/engine/jobs/job_progress.py
@@ -1,28 +1,8 @@
-import inspect
import logging
-from typing import Callable, List
logger = logging.getLogger(__name__)
-def job_progress_param_names(func: Callable) -> List[str]:
- """Names of the parameters of *func* that should receive a :class:`JobProgress`.
-
- A parameter qualifies when it is literally named ``job_progress`` or when its
- annotation refers to a ``JobProgress`` type. This single detection is shared
- by every injection site (queue worker, RunPod handler, direct FastAPI path)
- so they can never disagree about what counts as a progress parameter.
- """
- try:
- params = inspect.signature(func).parameters.values()
- except (TypeError, ValueError):
- return []
- return [
- p.name for p in params
- if p.name == "job_progress" or "JobProgress" in str(p.annotation)
- ]
-
-
class JobProgress:
"""Live progress handle for a running job.
diff --git a/apipod/engine/queue/job_queue.py b/apipod/engine/queue/job_queue.py
index ee9dd92..292e423 100644
--- a/apipod/engine/queue/job_queue.py
+++ b/apipod/engine/queue/job_queue.py
@@ -6,7 +6,7 @@
from apipod.engine.queue.job_store import JobStore
from apipod.engine.jobs.base_job import BaseJob, LocalJob, JOB_STATUS
-from apipod.engine.jobs.job_progress import job_progress_param_names
+from apipod.engine.signatures.analysis import job_progress_param_names
from apipod.engine.queue.job_queue_interface import JobQueueInterface
from apipod.engine.streaming.stream_producer import StreamProducer
diff --git a/apipod/engine/signatures/analysis.py b/apipod/engine/signatures/analysis.py
new file mode 100644
index 0000000..7f41d91
--- /dev/null
+++ b/apipod/engine/signatures/analysis.py
@@ -0,0 +1,47 @@
+"""
+Backend-neutral analysis of function signatures and return types.
+"""
+
+import inspect
+from collections.abc import AsyncIterator, Iterator
+from typing import Any, Callable, get_origin, get_type_hints
+
+
+def is_streaming_endpoint(func: Callable) -> bool:
+ """Backend-neutral: True if *func* is a generator or annotated as an Iterator.
+
+ A generator function (``yield``) or async generator function (``async yield``)
+ is always considered streaming. A regular function whose return annotation
+ is ``Iterator[...]`` / ``AsyncIterator[...]`` or any subclass is also detected.
+ """
+ target = inspect.unwrap(func)
+ if inspect.isgeneratorfunction(target) or inspect.isasyncgenfunction(target):
+ return True
+ try:
+ return_type = get_type_hints(target).get("return")
+ except Exception:
+ return False
+ if return_type is None:
+ return False
+ origin = get_origin(return_type) or return_type
+ return inspect.isclass(origin) and issubclass(origin, (Iterator, AsyncIterator))
+
+
+def is_injected_progress_param(param: inspect.Parameter) -> bool:
+ """True if the parameter is a framework-injected :class:`JobProgress`."""
+ return param.name == "job_progress" or "JobProgress" in str(param.annotation)
+
+
+def job_progress_param_names(func: Callable) -> list[str]:
+ """Names of the parameters of *func* that should receive a :class:`JobProgress`.
+
+ A parameter qualifies when it is literally named ``job_progress`` or when its
+ annotation refers to a ``JobProgress`` type. This single detection is shared
+ by every injection site (queue worker, RunPod handler, direct FastAPI path)
+ so they can never disagree about what counts as a progress parameter.
+ """
+ try:
+ params = inspect.signature(func).parameters.values()
+ except (TypeError, ValueError):
+ return []
+ return [p.name for p in params if is_injected_progress_param(p)]
From 1a813378d8f5a2a94be020496c73421ec39bc89a Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Tue, 16 Jun 2026 16:22:22 +0200
Subject: [PATCH 14/38] Replaced the orchestrator, provider, compute triplets
with a single command. apipod --simulate serverless-runpod. Introduced the
simulate and --direct flags for developer flexibility. Introduced a cert for
overwriting settings when managed by socaity
---
README.md | 320 ++++++-------------
apipod/api.py | 225 ++++++-------
apipod/cli.py | 214 ++++++++-----
apipod/common/constants.py | 5 -
apipod/common/settings.py | 24 +-
apipod/deploy/docker_template.j2 | 1 -
apipod/engine/backend/runpod/router.py | 19 +-
docs/{README_TECH.md => TECHNICAL_README.md} | 32 +-
test/test_config_resolution.py | 129 +++-----
test/test_core_functionalities.py | 2 +-
test/test_streaming.py | 14 +-
11 files changed, 432 insertions(+), 553 deletions(-)
rename docs/{README_TECH.md => TECHNICAL_README.md} (81%)
diff --git a/README.md b/README.md
index 36d645e..929ed03 100644
--- a/README.md
+++ b/README.md
@@ -8,15 +8,17 @@
- APIPod is the way for building and deploying AI services.
- Combining the developer experience of FastAPI with the power of Serverless GPU computing.
+ APIPod combines the developer experience of FastAPI with the power of Serverless GPU computing.
+ Write your service like FastAPI. Run it anywhere with a single command.
+ Think Vercel — but for AI services.
Why APIPod •
Installation •
Quick Start •
- Deployment
+ Develop & Test •
+ Build & Deploy
---
@@ -25,14 +27,16 @@
Building AI services is complex: file handling, long-running inference, job queues, deployment, scaling, and hosting provider choices all create friction at every step.
-**APIPod** solves this by standardizing the entire stack.
+**APIPod** eliminates that friction. It abstracts away the AI infrastructure stack so you can focus on your model. You write the service; [Socaity](https://www.socaity.ai) handles the deployment and scaling across any cloud.
-### 🚀 Highlights
-1. **Write Powerful APIs Instantly**: Built on top of FastAPI, it feels familiar but comes with batteries included for AI services.
-2. **Standardized I/O**: Painless handling of Images, Audio, and Video via [MediaToolkit](https://github.com/SocAIty/media-toolkit).
-3. **Automatic packaging**: The package can configure docker and deployment for you. No Cuda hell; the package knows compatible options.
-4. **Streamlined Deployment**: Deploy as a standard container or to serverless providers (**Socaity.ai** or **RunPod**) with zero configuration changes. Auth included.
-5. **Native SDK**: Built-in support for **Asynchronous Job Queues**, polling & and progress tracking via [fastSDK](https://github.com/SocAIty/fastSDK)
+### Highlights
+
+1. **Write once, run anywhere** — the same code runs in development, in serverless emulation, or on a real GPU cloud. Zero changes between environments.
+2. **Drop-in FastAPI** — if you know FastAPI, you already know APIPod. Built on top of it, with batteries included for AI.
+3. **Standardized I/O** — painless Images, Audio and Video via [media-toolkit](https://github.com/SocAIty/media-toolkit).
+4. **OpenAI-compatible schemas** — built-in request/response schemas for chat, completions, embeddings, TTS, transcription, image/video generation. OpenAI clients work out of the box.
+5. **Built-in job queue** — async jobs, polling and progress tracking, with no Celery/Redis/Kubernetes to wire up.
+6. **One-command packaging** — `apipod --build` generates your Dockerfile. No CUDA hell; APIPod picks compatible images.
## Installation
@@ -42,46 +46,41 @@ pip install apipod
## Quick Start
-### 1. Create your Service
-
-Zero-Hassle Migration: Replacing `FastAPI` with `APIPod` gives you instant access to all APIPod capablities..
+`APIPod` is a drop-in replacement for `FastAPI`. You get all of APIPod's capabilities with no migration cost.
```python
from apipod import APIPod, ImageFile
-# 1. Initialize APIPod (Drop-in replacement for FastAPI)
+# Drop-in replacement for FastAPI
app = APIPod()
-# 2. Define a standard endpoint (Synchronous)
+# A standard endpoint
@app.endpoint("/hello")
def hello(name: str):
return f"Hello {name}!"
-# 2. Use built-in media processing
-@app.endpoint("/process_image", queue_size=10)
+# Built-in media processing — uploads/URLs/base64 are parsed for you
+@app.endpoint("/process_image")
def process_image(image: ImageFile):
- # APIPod handles the file upload/parsing automatically
img_array = image.to_np_array()
-
# ... run your AI model here ...
-
return ImageFile().from_np_array(img_array)
-# 4. Run the server
if __name__ == "__main__":
app.start()
```
-### 2. Run Locally
+Run it and open `http://localhost:8000/docs` for the auto-generated Swagger UI.
+
```bash
python main.py
+# or
+apipod --start
```
-Visit `http://localhost:8000/docs` to see your auto-generated Swagger UI.
-## Features in Depth
+## Smart File Handling
-### 📁 Smart File Handling
-Forget about parsing `multipart/form-data`, `base64`, or `bytes`. APIPod integrates with **MediaToolkit** to handle files as objects.
+Forget about parsing `multipart/form-data`, `base64`, or `bytes`. APIPod integrates with **MediaToolkit** to handle files as objects. Whether the client sends a file upload, a URL, or a base64 string, your endpoint receives a ready-to-use object.
```python
from apipod import AudioFile
@@ -93,262 +92,143 @@ def transcribe(audio: AudioFile):
return {"transcription": "..."}
```
-### 🤖 Standardized AI schemas (OpenAI-compatible)
+## AI Services Streamlined (OpenAI-compatible)
-APIPod ships request/response schemas for the common AI payloads: chat, completions, embeddings, image/video/3D generation, vision, transcription, text-to-speech and voice cloning. Annotate one parameter with a request schema and APIPod handles validation, media parsing, response wrapping and streaming. The wire format mirrors the OpenAI API, so OpenAI-compatible clients work out of the box. `model` is optional everywhere — your service usually *is* the model.
-
-**Chat** — return a plain string (or a dict / full response object) and get a complete `chat.completion` envelope. For `"stream": true` requests just yield raw tokens: APIPod wraps each into a `ChatCompletionChunk` server-sent event — generating the chunk `id`, `created` and `object` out of the box — and emits the closing chunk plus the `[DONE]` sentinel:
+APIPod provides built-in request/response schemas for common AI tasks (chat, TTS, image gen, etc.) that are fully OpenAI-compatible. This allows you to focus on the model logic while APIPod handles the boilerplate of validation, media parsing, and streaming.
```python
-from apipod import APIPod
from apipod.common.schemas import ChatCompletionRequest
-app = APIPod()
-
@app.endpoint("/chat")
def chat(request: ChatCompletionRequest):
if request.stream:
# Yield plain tokens — APIPod wraps them into ChatCompletionChunk SSE events.
- return my_llm.stream(request.messages, temperature=request.temperature)
- answer = my_llm.generate(request.messages, temperature=request.temperature)
- return answer # auto-wrapped into a ChatCompletionResponse
+ return my_llm.stream(request.messages)
+ return my_llm.generate(request.messages) # auto-wrapped into ChatCompletionResponse
```
-**Text-to-speech** — return an `AudioFile`; clients get a JSON envelope with the audio as base64, or raw audio chunks when they send `"stream": true`:
+## Asynchronous Jobs & Scaling
-```python
-from apipod import AudioFile
-from apipod.common.schemas import SpeechRequest
-
-@app.endpoint("/audio/speech")
-def speak(request: SpeechRequest):
- # request.voice is a named voice (str) or an already-parsed AudioFile for cloning
- samples, rate = my_tts.synthesize(request.input, voice=request.voice)
- return AudioFile().from_np_array(samples, sample_rate=rate, audio_format="wav")
-```
-
-**Image generation** — media fields inside schemas (e.g. `request.image` for img2img) arrive as parsed media-toolkit objects, no matter if the client sent an upload, URL or base64:
+For long-running tasks, APIPod provides a built-in job queue and progress reporting. When configured for serverless or with a queue, endpoints automatically return a `job_id` and run in the background.
```python
-from apipod import ImageFile
-from apipod.common.schemas import ImageGenerationRequest
-
-@app.endpoint("/images/generations")
-def generate(request: ImageGenerationRequest):
- init_image = request.image.to_np_array() if request.image else None # ImageFile, ready to use
- result = my_model.generate(request.prompt, image=init_image, seed=request.seed)
- return ImageFile().from_np_array(result) # auto-wrapped into ImageGenerationResponse
+from apipod import JobProgress
+
+@app.post("/generate", queue_size=50)
+def generate(job_progress: JobProgress, prompt: str):
+ job_progress.set_status(0.1, "Initializing model...")
+ # ... heavy computation ...
+ job_progress.set_status(1.0, "Done!")
+ return "Generation Complete"
```
-Need extra parameters? Subclass a schema (`class MyRequest(SpeechRequest): style: str = "calm"`) — detection and wrapping keep working.
-
-### ☁️ Serverless Routing
-When deploying to serverless platforms like **RunPod**, standard web frameworks often fail because they lack the necessary routing logic for the platform's specific entry points. **APIPod** detects the environment and handles the routing automatically—no separate "handler" function required.
+* **Client:** Receives a `job_id` immediately.
+* **Server:** Processes the task in the background.
+* **SDK:** Automatically polls for status and result.
+## Develop, Test, and Simulate
-### 🔄 Scaling services, Asynchronous Jobs, Polling and Job Progress
-
-Let's say you want to serve your service to many users or you have a long-running task.
-Usually you need to set-up a load-balancer, kubernetes, brokers and a lot of other complicated stuff.
-If you deploy to socaity / runpod this is taken care of for you. No Dev-Ops for you.
-
-We allow you to emulate this behaviour for testing.
-
-For long-running tasks (e.g., inference of a large model), you don't want to block the HTTP request.
-Often you want to be able to give a progress bar or updates about the current task to the user. This is what job progress is for.
-It allows you to communicate a progress percentage and a status message to your user.
-
-
-
-1. **Setup test environment for serverless (Job Queue)**:
- ```python
- # Initialize with serverless compute on localhost to enable the local job queue
- app = APIPod(compute="serverless", provider="localhost")
- ```
-
-2. **Define Endpoint**:
- Use `@app.endpoint` (or `@app.post`). It automatically becomes a background task when a queue is configured.
- ```python
- @app.post("/generate", queue_size=50)
- def generate(job_progress: JobProgress, prompt: str):
- job_progress.set_status(0.1, "Initializing model...")
- # ... heavy computation ...
- job_progress.set_status(1.0, "Done!")
- return "Generation Complete"
- ```
-
- * **Client:** Receives a `job_id` immediately.
- * **Server:** Processes the task in the background.
- * **[SDK](https://github.com/SocAIty/fastSDK):** Automatically polls for status and result.
-
-3. **Opt-out**:
- If you want a standard synchronous endpoint even when queue is enabled:
- ```python
- @app.endpoint("/ping", use_queue=False)
- def ping():
- return "pong"
- ```
-
-
-# Deployment
-APIPod is designed to run anywhere by leveraging docker.
-
- Build & configure •
- Deploy
-
+Just say *how you want to run the service right now*.
-## Create & configure container
+### Development (default)
-All you need to do is run:
+Plain FastAPI. The fastest iteration loop.
```bash
-apipod build
+apipod --start
+# or simply
+python main.py
```
-This command creates the dockerfile for you, and select the correct docker template and cuda/cudnn versions and comes with ffmpeg installed.
-
-- For most users this already creates a sufficient solution.
-- However you are always free to create or customize the Dockerfile for your needs.
-
-Requirements:
-1. docker installed on your system.
-2. Depending on your setup a cuda/cudnn installation
+### Simulate a deployment
-### APIPod Configuration
-APIPod provides a flexible deployment configuration that allows developers to:
-- Run services locally for development
-- Deploy via the Socaity orchestration platform
-- Deploy directly to cloud providers
-- Choose between serverless or dedicated compute
+Before you ship, run your service exactly how it will behave in production — locally, with **no code changes**. `--simulate` takes an optional target string `{compute}-{provider}` (compute defaults to `serverless`).
-The configuration is controlled through a combination of:
-- orchestrator
-- compute
-- provider
-- region
-- CPU/GPU
-
-
-| Orchestrator | Compute | Provider | Resulting Backend |
-| ---------------- | ------------ | ----------- | --------------------------------- |
-| `socaity` | `dedicated` | `auto` | FastAPI |
-| `socaity` | `dedicated` | `localhost` | FastAPI + job queue *(test mode)* |
-| `socaity` | `dedicated` | `runpod` | FastAPI + load balancer *(Coming soon)* |
-| `socaity` | `dedicated` | `scaleway` | FastAPI *(Coming soon)* |
-| `socaity` | `dedicated` | `azure` | FastAPI *(Coming soon)* |
-| `socaity` | `serverless` | `auto` | RunPod router backend |
-| `socaity` | `serverless` | `localhost` | FastAPI + job queue *(test mode)* |
-| `socaity` | `serverless` | `runpod` | RunPod router backend |
-| `socaity` | `serverless` | `scaleway` | ❌ Not supported |
-| `socaity` | `serverless` | `azure` | ❌ Not supported |
-| `local` / `None` | `dedicated` | `localhost` | FastAPI |
-| `local` / `None` | `dedicated` | `runpod` | FastAPI |
-| `local` / `None` | `dedicated` | `scaleway` | FastAPI |
-| `local` / `None` | `dedicated` | `azure` | FastAPI |
-| `local` / `None` | `serverless` | `localhost` | FastAPI + job queue |
-| `local` / `None` | `serverless` | `runpod` | RunPod router backend |
-| `local` / `None` | `serverless` | `scaleway` | ❌ Not supported |
-| `local` / `None` | `serverless` | `azure` | ❌ Not supported |
-| `local` / `None` | `localhost` | `localhost` | FastAPI |
+```bash
+apipod --simulate # serverless emulation: FastAPI + local job queue
+apipod --simulate serverless # same as above
+apipod --simulate dedicated # plain FastAPI (dedicated compute)
+apipod --simulate serverless-runpod # emulate Socaity routing requests to RunPod
+apipod --simulate dedicated-azure # emulate a dedicated Azure deployment
+```
+If a provider has no serverless offering, APIPod warns and falls back to the job-queue emulation:
+```bash
+apipod --simulate serverless-azure
+# Warning: azure does not support serverless. Defaulting to FastAPI + Local Job Queue.
+```
+### Emulate a provider's native worker (`--direct`)
-### 🔄 Queue Backend Support
+By default Socaity is the orchestrator. `--direct` skips Socaity and runs the provider's **own** serverless backend locally — e.g. RunPod's serverless worker (requires the `runpod` package):
-APIPod supports multiple job queue backends to handle different deployment scenarios and scaling needs.
+```bash
+apipod --simulate serverless-runpod --direct
+```
-#### Available Backends
+### Configure from Python
-- **None** (default): Standard FastAPI behavior. No background jobs.
-
-- **Local Queue** (`local`): In-memory job queue using threading.
- - Perfect for local development and single-instance deployments
- - No external dependencies required
+The same intent can be set in code. Socaity **overrides** it with env vars once the service is actually managed by the platform, so what you test is what you ship.
-#### Configuration
+```python
+app = APIPod() # development (plain FastAPI)
+app = APIPod(simulate="serverless") # FastAPI + local job queue
+app = APIPod(simulate="serverless-runpod", direct=True) # RunPod native worker (local)
+```
- ```python
- # Job queues are automatically enabled based on your configuration.
- # For example, serverless + localhost enables a local job queue for testing:
- app = APIPod(compute="serverless", provider="localhost")
+## Build & Deploy
- # Or via environment variables
- import os
- os.environ["APIPOD_COMPUTE"] = "serverless"
- os.environ["APIPOD_PROVIDER"] = "localhost"
+### Build a container
- app = APIPod() # Uses environment config
- ```
+```bash
+apipod --build
+```
-### Troubleshooting
+This scans your project, picks a compatible base image (CUDA/cuDNN, ffmpeg included) and generates a `Dockerfile`. For most users this is all you need; advanced users can edit or write their own Dockerfile.
-You are always free to create or edit the Dockerfile for your needs.
-Depending on your OS, your machine or your project setup you might occur one of those issues:
-- Build scripts fails
-- You can't build the docker container.
+Requirements: Docker installed, plus a CUDA/cuDNN setup if your model needs the GPU.
-In this cases don't
-Advanced users can also configure or write the docker file for themselves
+### Deploy
-## Deploy to socaity
-Right after build you can deploy the service via the [socaity.ai](https://www.socaity.ai) dashboard.
-This is the simplest option.
+```bash
+apipod --deploy # coming soon
+apipod --deploy serverless
+apipod --deploy dedicated-azure
+```
-## Deploy to runpod.
-1. You will need to build the your docker image.
-2. Push your image to your dockerhub repository.
-3. Deploy on RunPod Serverless by using the runpod dashboard.
- * *APIPod acts as the handler, managing job inputs/outputs compatible with RunPod's API.*
+The managed `--deploy` command is on the roadmap. Today, build your container and deploy it through the [Socaity dashboard](https://www.socaity.ai) — the simplest path, with auth, scaling and routing handled for you.
-Make sure that the environment variables are set to the following: ```APIPOD_COMPUTE="serverless"``` and ```APIPOD_PROVIDER="runpod"```
+## Client SDK
+Generate a typed client for your service using the [fastSDK](https://github.com/SocAIty/fastSDK). It handles authentication, file uploads, and automatic polling for background jobs.
-## Debugging APIPod serverless
-You can configure your environment variables so that APIPod acts as if it were deployed on socaity.ai or on runpod.
```bash
-# Orchestrator
-ENV APIPOD_ORCHESTRATOR="local" # Options: "local" (default), "socaity"
-
-# Compute type
-ENV APIPOD_COMPUTE="serverless" # Options: "dedicated" (default), "serverless"
-
-# Infrastructure provider
-ENV APIPOD_PROVIDER="runpod" # Options: "localhost" (default), "auto", "runpod", "scaleway", "azure"
+fastsdk generate http://localhost:8009 -o myClient.py
```
-
-# Client SDK
-
-While you can use `curl` or `requests`, our [FastSDK](https://github.com/SocAIty/fastSDK) makes interacting with APIPod services feel like calling native Python functions.
-
```python
-# The SDK handles authentication, file uploads, and result polling
-# create a full working client stub
-create_sdk("https://localhost:8000", save_path="my_service.py")
-
-# Import the client. It will have a method for each of your service endpoints including all parameters and its default values.
-from my_service import awesome_client
-mySDK = awesome_client()
-mySDK.my_method(...)
+from myClient import myService
-# Blocks until the remote job is finished
-result = task.get_result()
+client = myService()
+client.text_to_speech("what a time to be alive")
```
-# Comparison
+## Comparison
| Feature | APIPod | FastAPI | Celery | Replicate/Cog |
| :--- | :---: | :---: | :---: | :---: |
-| **Setup Difficulty** | ⭐ Easy | ⭐ Easy | ⭐⭐⭐ Hard | ⭐⭐ Medium |
+| **Setup Difficulty** | Easy | Easy | Hard | Medium |
| **Async/Job Queue** | ✅ Built-in | ❌ Manual | ✅ Native | ✅ Native |
| **Serverless Ready** | ✅ Native | ❌ Manual | ❌ No | ✅ Native |
| **File Handling** | ✅ Standardized | ⚠️ Manual | ❌ Manual | ❌ Manual |
| **Router Support** | ✅ | ✅ | ❌ | ❌ |
+| **Multi-cloud** | ✅ | ❌ | ❌ | ❌ |
## Roadmap
+
+- `apipod --deploy` managed deployment command.
- MCP protocol support.
-- Improve async support.
---
diff --git a/apipod/api.py b/apipod/api.py
index e95f726..97f11a3 100644
--- a/apipod/api.py
+++ b/apipod/api.py
@@ -1,162 +1,133 @@
-from apipod.common import constants
-from apipod.common.settings import APIPOD_ORCHESTRATOR, APIPOD_COMPUTE, APIPOD_PROVIDER
-from apipod.engine.base_backend import _BaseBackend
-from apipod.engine.backend.runpod.router import SocaityRunpodRouter
+from typing import Optional, Tuple, Union
+
+from apipod.common.constants import COMPUTE, PROVIDER
+from apipod.common.settings import (
+ APIPOD_COMPUTE,
+ APIPOD_DIRECT,
+ APIPOD_PROVIDER,
+ APIPOD_SIMULATE,
+ IS_MANAGED_DEPLOYMENT,
+)
from apipod.engine.backend.fastapi.router import SocaityFastAPIRouter
-from apipod.engine.queue.job_queue_interface import JobQueueInterface
+from apipod.engine.backend.runpod.router import SocaityRunpodRouter
-from typing import Union
+# (backend_class, use_job_queue, runpod_simulate)
+_Resolution = Tuple[type, bool, bool]
+
+_NO_SERVERLESS = (PROVIDER.AZURE, PROVIDER.SCALEWAY)
def APIPod(
- orchestrator: Union[constants.ORCHESTRATOR, str, None] = None,
- compute: Union[constants.COMPUTE, str, None] = None,
- provider: Union[constants.PROVIDER, str, None] = None,
+ simulate: Union[str, None] = None,
+ direct: Union[bool, None] = None,
*args, **kwargs
-) -> Union[_BaseBackend, SocaityRunpodRouter, SocaityFastAPIRouter]:
+) -> Union[SocaityFastAPIRouter, SocaityRunpodRouter]:
"""
- Initialize an APIPod router with the appropriate backend based on the deployment configuration.
-
- The resulting backend is determined by the combination of orchestrator, compute, and provider:
-
- | Orchestrator | Compute | Provider | Backend |
- |------------- |----------- |---------- |--------------------------- |
- | socaity | dedicated | auto | FastAPI |
- | socaity | dedicated | localhost | FastAPI + job queue (test) |
- | socaity | dedicated | socaity | FastAPI + redis (prod) |
- | socaity | dedicated | runpod | Celery (planned) |
- | socaity | dedicated | scaleway | Celery (planned) |
- | socaity | dedicated | azure | Celery (planned) |
- | socaity | serverless | auto | RunPod router |
- | socaity | serverless | localhost | FastAPI + job queue (test) |
- | socaity | serverless | runpod | RunPod router |
- | socaity | serverless | scaleway | Not supported |
- | socaity | serverless | azure | Not supported |
- | local/None | dedicated | * | FastAPI |
- | local/None | serverless | localhost | FastAPI + job queue |
- | local/None | serverless | runpod | RunPod router |
- | local/None | serverless | scaleway | Not supported |
- | local/None | serverless | azure | Not supported |
+ Build the right backend for an APIPod service from a single *intent*.
+
+ Socaity is the implicit orchestrator, so you never wire infrastructure by hand.
+ You only pick how the service should run *locally*:
+
+ - **Development** (default, ``APIPod()``): plain FastAPI — the fastest loop.
+ - **Simulation** (``simulate="{compute}-{provider}"``): emulate a deployment
+ locally. The target collapses compute + provider, e.g. ``"serverless"``,
+ ``"serverless-runpod"``, ``"dedicated-azure"``. Compute defaults to
+ ``serverless``. ``direct=True`` bypasses Socaity to emulate the provider's
+ own serverless worker (currently RunPod).
+
+ In a **managed deployment** (``SOCAITY_DEPLOYMENT_CERT`` verified) ``simulate``
+ and ``direct`` are ignored: Socaity injects ``APIPOD_COMPUTE`` /
+ ``APIPOD_PROVIDER`` and the real backend is selected from them.
Args:
- orchestrator: "socaity" or "local" (default from env / local).
- compute: "dedicated" or "serverless" (default from env / dedicated).
- provider: "auto", "localhost", "socaity", "runpod", "scaleway", "azure" (default from env / localhost).
+ simulate: deployment target to emulate, ``"{compute}-{provider}"``.
+ ``None`` runs plain FastAPI for development.
+ direct: emulate the provider's native serverless worker instead of the
+ Socaity job-queue emulation. Only affects ``serverless-runpod``.
"""
- orchestrator = _resolve_enum(orchestrator, constants.ORCHESTRATOR, APIPOD_ORCHESTRATOR, constants.ORCHESTRATOR.LOCAL)
- compute = _resolve_enum(compute, constants.COMPUTE, APIPOD_COMPUTE, constants.COMPUTE.DEDICATED)
- provider = _resolve_enum(provider, constants.PROVIDER, APIPOD_PROVIDER, constants.PROVIDER.LOCALHOST)
+ if IS_MANAGED_DEPLOYMENT:
+ backend_class, use_job_queue, runpod_simulate = _resolve_managed()
+ else:
+ backend_class, use_job_queue, runpod_simulate = _resolve_intent(simulate, direct)
- backend_class, use_job_queue = _resolve_backend(orchestrator, compute, provider)
+ if backend_class is SocaityRunpodRouter:
+ return SocaityRunpodRouter(simulate=runpod_simulate, *args, **kwargs)
- custom_job_queue = kwargs.pop("job_queue", None)
- if custom_job_queue:
+ # FastAPI backend: a job queue (+ stream store) turns it into the serverless
+ # emulation; without one it is plain FastAPI. A deployment may inject its own.
+ job_queue = kwargs.pop("job_queue", None)
+ if job_queue is not None:
use_job_queue = True
- job_queue = custom_job_queue
- else:
+ elif use_job_queue:
from apipod.engine.queue.job_queue import JobQueue
- job_queue = JobQueue() if use_job_queue else None
-
- if backend_class == SocaityFastAPIRouter:
- # Streaming (serverless emulation): when a queue runs, the worker relays
- # streaming output into a stream store consumed via GET /stream/{job_id}.
- # Default to the in-memory LocalStreamStore; a deployment can inject its
- # own (e.g. Redis-backed) store. Plain FastAPI (no queue) streams directly
- # and uses no stream store.
- if use_job_queue and "stream_store" not in kwargs:
- kwargs["stream_store"] = _create_stream_store()
- return backend_class(job_queue=job_queue, *args, **kwargs)
- else:
- return backend_class(*args, **kwargs)
-
-
-def _resolve_enum(value, enum_cls, env_default, fallback):
- """Coerce a value into an enum member, falling back through env default and hard default."""
- if value is None:
- value = env_default
- if isinstance(value, str):
- try:
- return enum_cls(value)
- except ValueError:
- raise ValueError(f"Invalid {enum_cls.__name__} value: '{value}'. Choose from: {[e.value for e in enum_cls]}")
- if isinstance(value, enum_cls):
- return value
- return fallback
-
-
-def _resolve_backend(
- orchestrator: constants.ORCHESTRATOR,
- compute: constants.COMPUTE,
- provider: constants.PROVIDER,
-) -> tuple:
- """
- Apply the configuration matrix and return (backend_class, use_job_queue).
- Raises for unsupported or not-yet-implemented combinations.
- """
- _raise_if_unsupported(compute, provider)
+ job_queue = JobQueue()
- if orchestrator == constants.ORCHESTRATOR.SOCAITY:
- return _resolve_socaity(compute, provider)
+ if use_job_queue and "stream_store" not in kwargs:
+ kwargs["stream_store"] = _create_stream_store()
- return _resolve_local(compute, provider)
+ return SocaityFastAPIRouter(job_queue=job_queue, *args, **kwargs)
-def _raise_if_unsupported(compute: constants.COMPUTE, provider: constants.PROVIDER):
- unsupported = {
- (constants.COMPUTE.SERVERLESS, constants.PROVIDER.SCALEWAY),
- (constants.COMPUTE.SERVERLESS, constants.PROVIDER.AZURE),
- }
- if (compute, provider) in unsupported:
- raise NotImplementedError(
- f"Serverless compute on {provider.value} is not supported. "
- f"Use provider='runpod' for serverless or switch to dedicated compute."
- )
+def _resolve_intent(simulate: Optional[str], direct: Optional[bool]) -> _Resolution:
+ """Resolve a local run (development or simulation) into a backend selection."""
+ target = APIPOD_SIMULATE if simulate is None else simulate
+ # Development: no simulation requested -> plain FastAPI.
+ if simulate is None and not APIPOD_SIMULATE:
+ return SocaityFastAPIRouter, False, False
-def _resolve_socaity(compute: constants.COMPUTE, provider: constants.PROVIDER) -> tuple:
- if compute == constants.COMPUTE.DEDICATED:
- if provider == constants.PROVIDER.SOCAITY:
- return SocaityFastAPIRouter, True
+ direct = APIPOD_DIRECT if direct is None else bool(direct)
+ compute, provider = _parse_target(target)
- if provider in (constants.PROVIDER.RUNPOD, constants.PROVIDER.SCALEWAY, constants.PROVIDER.AZURE):
- raise NotImplementedError(
- f"Celery backend for socaity + dedicated + {provider.value} is planned but not yet available."
- )
- if provider == constants.PROVIDER.LOCALHOST:
- return SocaityFastAPIRouter, True
- # auto or any other -> FastAPI without queue
- return SocaityFastAPIRouter, False
+ if compute is COMPUTE.DEDICATED:
+ # "Standard FastAPI"; with a named provider it emulates a direct client.
+ return SocaityFastAPIRouter, False, False
# serverless
- if provider == constants.PROVIDER.LOCALHOST:
- return SocaityFastAPIRouter, True
- # auto or runpod -> RunPod router
- return SocaityRunpodRouter, False
+ if provider in _NO_SERVERLESS:
+ print(f"Warning: {provider.value} does not support serverless. "
+ f"Defaulting to FastAPI + Local Job Queue.")
+ return SocaityFastAPIRouter, True, False
+ if provider is PROVIDER.RUNPOD and direct:
+ # Emulate RunPod's native serverless worker locally.
+ return SocaityRunpodRouter, False, True
-def _resolve_local(compute: constants.COMPUTE, provider: constants.PROVIDER) -> tuple:
- if compute == constants.COMPUTE.DEDICATED:
- return SocaityFastAPIRouter, False
+ # Default serverless: Socaity emulation = FastAPI + Local Job Queue.
+ return SocaityFastAPIRouter, True, False
- # serverless
- if provider == constants.PROVIDER.LOCALHOST:
- return SocaityFastAPIRouter, True
- if provider == constants.PROVIDER.RUNPOD:
- return SocaityRunpodRouter, False
- # auto -> RunPod router (same default as socaity serverless auto)
- if provider == constants.PROVIDER.AUTO:
- return SocaityRunpodRouter, False
- raise NotImplementedError(f"Unsupported configuration: local + serverless + {provider.value}")
+def _resolve_managed() -> _Resolution:
+ """Pick the real production backend from the env vars Socaity injects."""
+ compute = COMPUTE(APIPOD_COMPUTE)
+ provider = PROVIDER(APIPOD_PROVIDER)
+ if compute is COMPUTE.SERVERLESS and provider is PROVIDER.RUNPOD:
+ return SocaityRunpodRouter, False, False # real serverless worker
+ if compute is COMPUTE.SERVERLESS:
+ return SocaityFastAPIRouter, True, False
+ return SocaityFastAPIRouter, False, False # dedicated (queue injected if needed)
-def _create_job_queue() -> JobQueueInterface:
- from apipod.engine.queue.job_queue import JobQueue
- return JobQueue()
+def _parse_target(target: str) -> Tuple[COMPUTE, Optional[PROVIDER]]:
+ """Parse a ``"{compute}-{provider}"`` target. Provider is optional; compute defaults to serverless."""
+ if not target:
+ return COMPUTE.SERVERLESS, None
+
+ compute_str, _, provider_str = target.partition("-")
+ try:
+ compute = COMPUTE(compute_str)
+ except ValueError:
+ raise ValueError(f"Invalid compute '{compute_str}'. Choose from: {[c.value for c in COMPUTE]}")
+
+ if not provider_str:
+ return compute, None
+ try:
+ return compute, PROVIDER(provider_str)
+ except ValueError:
+ raise ValueError(f"Invalid provider '{provider_str}'. Choose from: {[p.value for p in PROVIDER]}")
def _create_stream_store():
from apipod.engine.streaming.local_stream_store import LocalStreamStore
-
return LocalStreamStore()
diff --git a/apipod/cli.py b/apipod/cli.py
index b2fecaf..c614019 100644
--- a/apipod/cli.py
+++ b/apipod/cli.py
@@ -1,10 +1,11 @@
import argparse
+import importlib.util
+import os
import sys
from pathlib import Path
from typing import Optional
from apipod.deploy.deployment_manager import DeploymentManager
-from apipod.common.constants import ORCHESTRATOR, COMPUTE, PROVIDER
def input_yes_no(question: str, default: bool = True) -> bool:
@@ -21,6 +22,13 @@ def input_yes_no(question: str, default: bool = True) -> bool:
sys.stdout.write("Please respond with 'yes' or 'no' (or 'y'/'n').\n")
+def _parse_bool(value) -> Optional[bool]:
+ """Coerce a CLI flag value into a bool. ``None`` (flag absent) stays ``None``."""
+ if value is None or isinstance(value, bool):
+ return value
+ return str(value).strip().lower() in ("1", "true", "yes", "y")
+
+
def select_base_image(manager: DeploymentManager, config_data: dict) -> str:
"""Interactive base image selection process."""
recommended_image = manager.recommend_image(config_data)
@@ -64,7 +72,7 @@ def perform_scan():
else:
print("Scanning project...")
return manager.scan()
-
+
if manager.config_exists:
if not input_yes_no(f"Found {manager.config_path.name} in {manager.config_path.parent}/. Overwrite?"):
return manager.load_config()
@@ -99,40 +107,32 @@ def run_build(args):
target_file = None
if args.build is not None and args.build is not True:
target_file = args.build
-
+
target_path = Path(target_file)
if not target_path.exists():
print(f"Error: Target file '{target_file}' does not exist.")
return
-
+
if not target_path.is_file():
print(f"Error: '{target_file}' is not a file.")
return
-
+
if not target_path.suffix == '.py':
print(f"Warning: '{target_file}' is not a Python file (.py)")
if not input_yes_no("Continue anyway?", default=False):
return
-
+
print(f"Using target file: {target_file}")
if manager.dockerfile_exists and not input_yes_no("Deployment config DOCKERFILE exists. Overwrite your deployment config?"):
print("Aborting build configuration.")
return
- should_create_dockerfile = True
-
config_data = get_or_create_config(manager, target_file)
if not config_data:
print("Error: Failed to obtain configuration.")
return
- config_data["orchestrator"] = args.orchestrator
- config_data["compute"] = args.compute
- config_data["provider"] = args.provider
- if args.region:
- config_data["region"] = args.region
-
service_title = config_data.get("title", "apipod-service")
final_image = select_base_image(manager, config_data)
@@ -146,106 +146,156 @@ def run_build(args):
print("Please configure dependencies and try again.")
return
- if should_create_dockerfile:
- print("Generating Dockerfile...")
- dockerfile_content = manager.render_dockerfile(final_image, config_data)
- manager.write_dockerfile(dockerfile_content)
+ print("Generating Dockerfile...")
+ dockerfile_content = manager.render_dockerfile(final_image, config_data)
+ manager.write_dockerfile(dockerfile_content)
if input_yes_no(f"Build the application now using docker? (Tag: {service_title})"):
manager.build_docker_image(service_title)
-def run_start(args):
- """Start an APIPod service with the given configuration."""
- from apipod import APIPod
+def _resolve_entrypoint(manager: DeploymentManager, entrypoint: Optional[str]) -> str:
+ """Return the service entrypoint file, scanning the project when not provided."""
+ if entrypoint:
+ return entrypoint
+
+ config = manager.load_config() if manager.config_exists else None
+ if not config:
+ print("No apipod.json found. Scanning project to locate the entrypoint...")
+ config = manager.scan()
+ manager.save_config(config)
+ return config.get("entrypoint", "main.py")
- app = APIPod(
- orchestrator=args.orchestrator,
- compute=args.compute,
- provider=args.provider,
- )
+
+def _load_app(entrypoint: str):
+ """Import the entrypoint module and return its APIPod application instance."""
+ from apipod.engine.base_backend import _BaseBackend
+
+ path = Path(entrypoint).resolve()
+ if not path.exists():
+ raise FileNotFoundError(f"Entrypoint '{entrypoint}' not found.")
+
+ spec = importlib.util.spec_from_file_location("apipod_entrypoint", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+
+ app = getattr(module, "app", None)
+ if isinstance(app, _BaseBackend):
+ return app
+ for value in vars(module).values(): # fall back to the first app in the module
+ if isinstance(value, _BaseBackend):
+ return value
+ raise RuntimeError(f"No APIPod app found in '{entrypoint}'. Expected an APIPod() instance.")
+
+
+def _run_service(args, simulate: Optional[str], direct: Optional[bool]):
+ """Resolve the entrypoint, apply the run intent via env vars, and start the app."""
+ # The user's service calls APIPod() with no args; it reads the intent from env.
+ # Set the env BEFORE importing the entrypoint (which imports apipod settings).
+ if simulate is not None:
+ os.environ["APIPOD_SIMULATE"] = simulate
+ if direct is not None:
+ os.environ["APIPOD_DIRECT"] = "true" if direct else "false"
+
+ manager = DeploymentManager()
+ entrypoint = _resolve_entrypoint(manager, args.entrypoint)
+ app = _load_app(entrypoint)
port = args.port or 8000
host = args.host or "0.0.0.0"
- print(f"Starting APIPod (orchestrator={args.orchestrator}, compute={args.compute}, provider={args.provider})")
+ mode = "development" if simulate is None else f"simulation '{simulate}'{' --direct' if direct else ''}"
+ print(f"Starting APIPod ({mode}) from {entrypoint}")
app.start(port=port, host=host)
+def run_start(args):
+ """Run the service locally for development (plain FastAPI)."""
+ _run_service(args, simulate=None, direct=None)
+
+
+def run_simulate(args):
+ """Run the service locally while emulating a deployment target."""
+ target = args.simulate or "serverless" # bare --simulate defaults to serverless
+ _run_service(args, simulate=target, direct=_parse_bool(args.direct))
+
+
+def run_deploy(args):
+ """Placeholder for the upcoming managed deployment command."""
+ target = args.deploy or "serverless"
+ print(
+ f"`apipod --deploy {target}` is not available yet.\n"
+ "Deploy through the Socaity dashboard for now: https://www.socaity.ai\n"
+ "Tip: validate the target locally first with `apipod --simulate "
+ f"{target}`."
+ )
+
+
def main():
"""Main entry point for the APIPod CLI."""
- orchestrator_choices = [e.value for e in ORCHESTRATOR]
- compute_choices = [e.value for e in COMPUTE]
- provider_choices = [e.value for e in PROVIDER]
-
parser = argparse.ArgumentParser(
- description="APIPod CLI - Build and deploy AI service containers",
+ description="APIPod CLI - build, simulate and deploy AI services",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
+ apipod --start Run locally for development (FastAPI)
+ apipod --simulate Emulate a serverless deployment (FastAPI + job queue)
+ apipod --simulate serverless-runpod Emulate Socaity managed deploy to RunPod
+ apipod --simulate serverless-runpod --direct Emulate RunPod's native worker directly
+ apipod --simulate dedicated-azure Emulate a dedicated Azure deployment
apipod --scan Scan project and generate apipod.json
- apipod --build Build container using current directory
- apipod --build ./main.py Build container with specific entry file
- apipod --build --provider runpod Build for RunPod deployment
- apipod --start Start service locally
- apipod --start --compute serverless Start in serverless emulation mode
- apipod --start --orchestrator socaity Start with Socaity orchestrator
+ apipod --build Build the deployment container
+ apipod --deploy Deploy via Socaity (coming soon)
"""
)
- parser.add_argument(
- "--build",
- nargs="?",
- const=True,
- metavar="FILE",
- help="Build the service container. Optionally specify a target Python file (e.g., --build main.py)"
- )
- parser.add_argument(
- "--scan",
- action="store_true",
- help="Scan project and generate apipod.json configuration file"
- )
parser.add_argument(
"--start",
action="store_true",
- help="Start the APIPod service locally"
+ help="Run the service locally for development (plain FastAPI)."
)
-
- config_group = parser.add_argument_group("deployment configuration")
- config_group.add_argument(
- "--orchestrator",
- choices=orchestrator_choices,
- default="local",
- help=f"Orchestration platform (default: local). Options: {', '.join(orchestrator_choices)}"
+ parser.add_argument(
+ "--simulate",
+ nargs="?",
+ const="",
+ metavar="TARGET",
+ help="Emulate a deployment locally. Optional target '{compute}-{provider}' "
+ "(e.g. serverless-runpod, dedicated-azure). Defaults to 'serverless'."
)
- config_group.add_argument(
- "--compute",
- choices=compute_choices,
- default="dedicated",
- help=f"Compute type (default: dedicated). Options: {', '.join(compute_choices)}"
+ parser.add_argument(
+ "--direct",
+ nargs="?",
+ const=True,
+ default=None,
+ metavar="BOOL",
+ help="Emulate the provider's native serverless worker instead of Socaity's job queue."
)
- config_group.add_argument(
- "--provider",
- choices=provider_choices,
- default="localhost",
- help=f"Infrastructure provider (default: localhost). Options: {', '.join(provider_choices)}"
+ parser.add_argument(
+ "--deploy",
+ nargs="?",
+ const="",
+ metavar="TARGET",
+ help="Deploy the service via Socaity (coming soon)."
)
- config_group.add_argument(
- "--region",
- default=None,
- help="Deployment region (provider-specific, e.g. 'us-east-1')"
+ parser.add_argument(
+ "--scan",
+ action="store_true",
+ help="Scan project and generate apipod.json configuration file."
)
- config_group.add_argument(
- "--host",
- default=None,
- help="Host to bind to (default: 0.0.0.0)"
+ parser.add_argument(
+ "--build",
+ nargs="?",
+ const=True,
+ metavar="FILE",
+ help="Build the service container. Optionally specify a target Python file."
)
- config_group.add_argument(
- "--port",
- type=int,
+ parser.add_argument(
+ "--entrypoint",
default=None,
- help="Port to bind to (default: 8000)"
+ help="Service entrypoint file for --start / --simulate (auto-detected if omitted)."
)
+ parser.add_argument("--host", default=None, help="Host to bind to (default: 0.0.0.0).")
+ parser.add_argument("--port", type=int, default=None, help="Port to bind to (default: 8000).")
args = parser.parse_args()
@@ -253,6 +303,10 @@ def main():
run_scan()
elif args.build is not None:
run_build(args)
+ elif args.deploy is not None:
+ run_deploy(args)
+ elif args.simulate is not None:
+ run_simulate(args)
elif args.start:
run_start(args)
else:
diff --git a/apipod/common/constants.py b/apipod/common/constants.py
index 59f9bf8..62936a1 100644
--- a/apipod/common/constants.py
+++ b/apipod/common/constants.py
@@ -1,11 +1,6 @@
from enum import Enum
-class ORCHESTRATOR(Enum):
- SOCAITY = "socaity"
- LOCAL = "local"
-
-
class COMPUTE(Enum):
DEDICATED = "dedicated"
SERVERLESS = "serverless"
diff --git a/apipod/common/settings.py b/apipod/common/settings.py
index 4dcf34d..ba23eeb 100644
--- a/apipod/common/settings.py
+++ b/apipod/common/settings.py
@@ -1,12 +1,17 @@
-import sys
from os import environ
-from apipod.common.constants import ORCHESTRATOR, COMPUTE, PROVIDER
+from apipod.common.constants import COMPUTE, PROVIDER
-APIPOD_ORCHESTRATOR = environ.get("APIPOD_ORCHESTRATOR", ORCHESTRATOR.LOCAL)
-APIPOD_COMPUTE = environ.get("APIPOD_COMPUTE", COMPUTE.DEDICATED)
-APIPOD_PROVIDER = environ.get("APIPOD_PROVIDER", PROVIDER.LOCALHOST)
+# Deployment target. Socaity overwrites these env vars in a managed deployment so
+# the right backend is selected in production (e.g. serverless + runpod).
+APIPOD_COMPUTE = environ.get("APIPOD_COMPUTE", COMPUTE.DEDICATED.value)
+APIPOD_PROVIDER = environ.get("APIPOD_PROVIDER", PROVIDER.LOCALHOST.value)
APIPOD_REGION = environ.get("APIPOD_REGION", "")
+# Local simulation intent (ignored in managed deployments). Empty = development.
+# Target string is "{compute}-{provider}", e.g. "serverless-runpod".
+APIPOD_SIMULATE = environ.get("APIPOD_SIMULATE", "")
+APIPOD_DIRECT = environ.get("APIPOD_DIRECT", "").strip().lower() in ("1", "true", "yes")
+
APIPOD_HOST = environ.get("APIPOD_HOST", "0.0.0.0")
APIPOD_PORT = int(environ.get("APIPOD_PORT", 8000))
@@ -14,6 +19,9 @@
DEFAULT_DATE_TIME_FORMAT = environ.get("FTAPI_DATETIME_FORMAT", '%Y-%m-%dT%H:%M:%S.%f%z')
-if (APIPOD_ORCHESTRATOR == ORCHESTRATOR.SOCAITY or APIPOD_ORCHESTRATOR == ORCHESTRATOR.LOCAL) and APIPOD_PROVIDER == PROVIDER.RUNPOD and APIPOD_COMPUTE == COMPUTE.SERVERLESS:
- sys.argv.extend(['rp_serve_api', '1'])
- sys.argv.extend(['--rp_serve_api', '1'])
+# Socaity deployment certificate (SHA1 of a shared secret) for detecting a managed
+# deployment. When verified, simulate/direct are ignored and the backend is chosen
+# from APIPOD_COMPUTE / APIPOD_PROVIDER.
+SOCAITY_DEPLOYMENT_CERT = environ.get("SOCAITY_DEPLOYMENT_CERT", "")
+_EXPECTED_CERT_HASH = "7b35ca9da2f0c280d48f66c780a0a0d5d3f8ad8a"
+IS_MANAGED_DEPLOYMENT = SOCAITY_DEPLOYMENT_CERT == _EXPECTED_CERT_HASH
diff --git a/apipod/deploy/docker_template.j2 b/apipod/deploy/docker_template.j2
index a1689d2..420eb7d 100644
--- a/apipod/deploy/docker_template.j2
+++ b/apipod/deploy/docker_template.j2
@@ -58,7 +58,6 @@ COPY . .
RUN pip install --no-cache-dir .
# Configuration for APIPod deployment
-ENV APIPOD_ORCHESTRATOR="{{ orchestrator | default('local') }}"
ENV APIPOD_COMPUTE="{{ compute | default('serverless') }}"
ENV APIPOD_PROVIDER="{{ provider | default('runpod') }}"
ENV APIPOD_HOST="0.0.0.0"
diff --git a/apipod/engine/backend/runpod/router.py b/apipod/engine/backend/runpod/router.py
index 07f35ee..b9e0a73 100644
--- a/apipod/engine/backend/runpod/router.py
+++ b/apipod/engine/backend/runpod/router.py
@@ -23,7 +23,7 @@
from apipod.engine.streaming.stream_serializer import as_sync_iter, encode_chunk, is_streaming_result
from apipod.engine.utils import normalize_name
-from apipod.common.settings import APIPOD_PROVIDER, APIPOD_PORT
+from apipod.common.settings import APIPOD_PORT
from media_toolkit import AudioFile, VideoFile
@@ -32,9 +32,12 @@ class SocaityRunpodRouter(_BaseBackend, _BaseFileHandlingMixin):
Adds routing functionality for the runpod serverless framework.
Provides enhanced file handling and conversion capabilities.
"""
- def __init__(self, title: str = "APIPod for ", summary: str = None, *args, **kwargs):
+ def __init__(self, title: str = "APIPod for ", summary: str = None, simulate: bool = False, *args, **kwargs):
super().__init__(title=title, summary=summary, *args, **kwargs)
+ # When True, start() runs RunPod's local API emulator instead of the real
+ # serverless worker (set by APIPod(simulate="serverless-runpod", direct=True)).
+ self.simulate = simulate
self.routes = {} # routes are organized like {"ROUTE_NAME": "ROUTE_FUNCTION"}
self.add_standard_routes()
@@ -405,13 +408,13 @@ def minimal_func():
return schema
- def start(self, port: int = APIPOD_PORT, provider: Union[constants.PROVIDER, str, None] = None, *args, **kwargs):
- if provider is None:
- provider = APIPOD_PROVIDER
- if isinstance(provider, str):
- provider = constants.PROVIDER(provider)
+ def start(self, port: int = APIPOD_PORT, **kwargs):
+ """Start the RunPod worker.
- if provider == constants.PROVIDER.LOCALHOST:
+ In simulation (``APIPod(simulate="serverless-runpod", direct=True)``) RunPod's
+ local API emulator is used. In a managed deployment the real serverless worker runs.
+ """
+ if self.simulate:
self.start_runpod_serverless_localhost(port=port)
else:
import runpod.serverless
diff --git a/docs/README_TECH.md b/docs/TECHNICAL_README.md
similarity index 81%
rename from docs/README_TECH.md
rename to docs/TECHNICAL_README.md
index 85ba046..24dd744 100644
--- a/docs/README_TECH.md
+++ b/docs/TECHNICAL_README.md
@@ -18,10 +18,10 @@ It sits in an ecosystem of three packages:
```
apipod/
-├── api.py # APIPod() factory: resolves config → backend instance
+├── api.py # APIPod() factory: resolves intent → backend instance
├── common/
-│ ├── settings.py # Env-driven config (APIPOD_ORCHESTRATOR / _COMPUTE / _PROVIDER, host, port)
-│ ├── constants.py # Enums: ORCHESTRATOR, COMPUTE, PROVIDER, SERVER_HEALTH
+│ ├── settings.py # Env-driven config (APIPOD_SIMULATE / _DIRECT / _COMPUTE / _PROVIDER, cert, host, port)
+│ ├── constants.py # Enums: COMPUTE, PROVIDER, SERVER_HEALTH
│ └── schemas/
│ ├── schemas.py # Standardized request/response models (OpenAI-compatible shapes)
│ └── media_files.py # FileModel + typed variants: pydantic mirrors of media-toolkit files
@@ -42,19 +42,31 @@ apipod/
## Core principles
1. **One decorator, many execution modes.** Developers only write `@app.endpoint(...)`. The router inspects the function and decides how to run it. They never choose a "mode" explicitly — the signature is the contract.
-2. **Deployment is configuration, not code.** The same service file runs as a plain FastAPI app, a queued FastAPI app, or a RunPod serverless worker. `APIPod()` is a factory that picks the backend from `orchestrator` / `compute` / `provider` (constructor args or `APIPOD_*` env vars).
+2. **Deployment is intent, not code.** The same service file runs as a plain FastAPI app, a queued FastAPI app, or a RunPod serverless worker. `APIPod()` is a factory that picks the backend from a single *intent* (`simulate` / `direct`) locally, or from the `APIPOD_COMPUTE` / `APIPOD_PROVIDER` env vars Socaity injects in a managed deployment (see [Orchestrator, compute, provider](#orchestrator-compute-provider-and-simulate)).
3. **Media files are objects, not bytes.** Endpoint authors annotate parameters with media-toolkit types (`ImageFile`, `AudioFile`, …) and receive parsed, ready-to-use objects — regardless of whether the client sent a multipart upload, a URL, or base64.
4. **Standardized, OpenAI-compatible schemas.** Common AI payloads (chat, completion, embeddings, image/video/audio/3D generation, vision) have canonical pydantic schemas whose wire format mirrors the OpenAI API. The shape is provider-agnostic: any model can serve them; clients written against OpenAI-compatible tooling work without translation.
5. **Long-running work returns a job, not a blocked connection.** With a queue configured, endpoints return a `JobResult` immediately; clients poll `/status/{job_id}` (fastSDK does this automatically) and can receive progress updates via `JobProgress`.
-## Backend resolution
+## Orchestrator, compute, provider, and simulate
+
+Three concepts describe *where and how* a service runs. Historically developers had to set all three; they no longer do. Understanding them still helps when reading the code.
+
+- **Orchestrator** — *who routes and queues requests*. There is exactly one: **Socaity**. It is the implicit orchestrator that fronts the service, distributes jobs, and handles scaling/auth. It is no longer a flag. `--direct` is the one escape hatch: it *skips* Socaity to talk to a provider's native backend instead (e.g. RunPod's own serverless worker).
+- **Compute** (`COMPUTE`) — *the shape of the machine*: `serverless` (scale-to-zero, job-queue semantics) or `dedicated` (an always-on box, plain request/response).
+- **Provider** (`PROVIDER`) — *the cloud the compute lives on*: `runpod`, `azure`, `scaleway`, `localhost`, … Not every provider supports every compute (e.g. Azure has no serverless; APIPod warns and falls back to the job-queue emulation).
-`APIPod()` in `api.py` is not a class — it is a factory. It resolves the `(orchestrator, compute, provider)` triple (explicit args → env vars → defaults) against a support matrix and returns one of:
+A developer never assembles this matrix by hand. They express a single **intent**:
+
+- **Development** (`APIPod()` / `apipod --start`) — plain FastAPI, the fastest loop.
+- **Simulation** (`APIPod(simulate="{compute}-{provider}")` / `apipod --simulate ...`) — emulate a deployment **locally**, no code changes. The target string collapses compute + provider (`serverless`, `serverless-runpod`, `dedicated-azure`); compute defaults to `serverless`. `direct=True` emulates the provider's native worker instead of the Socaity queue.
+- **Managed deployment** — when the service runs on the platform, Socaity sets `SOCAITY_DEPLOYMENT_CERT` (SHA1 of a shared secret). When that cert verifies (`IS_MANAGED_DEPLOYMENT`), `simulate`/`direct` are ignored and the **real** backend is selected from the `APIPOD_COMPUTE` / `APIPOD_PROVIDER` env vars Socaity injects — so the serverless-RunPod path runs the *real* worker, not the emulator.
+
+## Backend resolution
-- **`SocaityFastAPIRouter`** — an `APIRouter` subclass bound to a `FastAPI` app. Used for dedicated compute and for local serverless emulation (then paired with an in-memory `JobQueue` plus a background worker thread started via the app lifespan).
-- **`SocaityRunpodRouter`** — a path-based dispatcher for RunPod serverless. There is no HTTP layer: RunPod delivers a JSON job whose `input.path` selects the registered function; the router converts files, injects `JobProgress`, executes, and returns a serialized `JobResult` (or a generator for streaming). It can also synthesize an OpenAPI schema by replaying the FastAPI signature conversion, so fastSDK clients can be generated against serverless deployments too.
+`APIPod()` in `api.py` is not a class — it is a factory. It resolves the intent (managed → `_resolve_managed`, otherwise → `_resolve_intent`) into a `(backend_class, use_job_queue, runpod_simulate)` triple and returns one of:
-The full configuration matrix is documented in the main README and in the `APIPod()` docstring.
+- **`SocaityFastAPIRouter`** — an `APIRouter` subclass bound to a `FastAPI` app. Used for development and dedicated compute (no queue), and for the serverless emulation (paired with an in-memory `JobQueue` + `LocalStreamStore` and a background worker thread started via the app lifespan).
+- **`SocaityRunpodRouter`** — a path-based dispatcher for RunPod serverless. There is no HTTP layer: RunPod delivers a JSON job whose `input.path` selects the registered function; the router converts files, injects `JobProgress`, executes, and returns a serialized `JobResult` (or a generator for streaming). It can also synthesize an OpenAPI schema by replaying the FastAPI signature conversion, so fastSDK clients can be generated against serverless deployments too. Its `simulate` flag chooses between RunPod's local API emulator (`apipod --simulate serverless-runpod --direct`) and the real worker (managed deployment).
## The endpoint pipeline (FastAPI backend)
@@ -126,7 +138,7 @@ The **stream store** is the pluggable backend that buffers a job's chunks betwee
### Deployment
-`apipod build` (see `deploy/`) scans the project (entrypoint, dependencies, CUDA requirements) and generates a Dockerfile from compatible templates. The resulting container runs unchanged on dedicated hosts, on socaity.ai, or on RunPod serverless — only the `APIPOD_*` env vars differ.
+`apipod --build` (see `deploy/`) scans the project (entrypoint, dependencies, CUDA requirements) and generates a Dockerfile from compatible templates. The resulting container runs unchanged on dedicated hosts, on socaity.ai, or on RunPod serverless — only the env vars differ: Socaity injects `SOCAITY_DEPLOYMENT_CERT` plus `APIPOD_COMPUTE` / `APIPOD_PROVIDER` to select the real backend, while locally you drive the same paths with `APIPOD_SIMULATE` / `APIPOD_DIRECT` (set for you by `apipod --simulate`).
## Request lifecycle, end to end
diff --git a/test/test_config_resolution.py b/test/test_config_resolution.py
index 269c42a..0b534b0 100644
--- a/test/test_config_resolution.py
+++ b/test/test_config_resolution.py
@@ -1,117 +1,74 @@
from apipod import APIPod
-from apipod.common import constants
from apipod.engine.backend.fastapi.router import SocaityFastAPIRouter
from apipod.engine.backend.runpod.router import SocaityRunpodRouter
-import os
-def test_socaity_dedicated_auto():
- # socaity | dedicated | auto -> FastAPI
- app = APIPod(orchestrator="socaity", compute="dedicated", provider="auto")
+def test_development_default():
+ # APIPod() -> plain FastAPI, no job queue.
+ app = APIPod()
assert isinstance(app, SocaityFastAPIRouter)
assert app.job_queue is None
-def test_socaity_dedicated_localhost():
- # socaity | dedicated | localhost -> FastAPI + job queue (test mode)
- app = APIPod(orchestrator="socaity", compute="dedicated", provider="localhost")
- assert isinstance(app, SocaityFastAPIRouter)
- assert app.job_queue is not None
+def test_simulate_serverless_default():
+ # simulate (bare / "serverless") -> FastAPI + Local Job Queue.
+ for target in ("", "serverless"):
+ app = APIPod(simulate=target)
+ assert isinstance(app, SocaityFastAPIRouter)
+ assert app.job_queue is not None
-def test_socaity_dedicated_planned_celery():
- # socaity | dedicated | runpod/scaleway/azure -> Celery (planned)
- for provider in ["runpod", "scaleway", "azure"]:
- try:
- APIPod(orchestrator="socaity", compute="dedicated", provider=provider)
- assert False
- except NotImplementedError as e:
- assert "planned" in str(e)
+def test_simulate_dedicated():
+ # dedicated -> Standard FastAPI (no queue).
+ app = APIPod(simulate="dedicated")
+ assert isinstance(app, SocaityFastAPIRouter)
+ assert app.job_queue is None
-def test_socaity_serverless_auto():
- # socaity | serverless | auto -> RunPod router
- app = APIPod(orchestrator="socaity", compute="serverless", provider="auto")
- assert isinstance(app, SocaityRunpodRouter)
+def test_simulate_dedicated_azure():
+ # dedicated-azure -> FastAPI (direct client), no queue.
+ app = APIPod(simulate="dedicated-azure")
+ assert isinstance(app, SocaityFastAPIRouter)
+ assert app.job_queue is None
-def test_socaity_serverless_localhost():
- # socaity | serverless | localhost -> FastAPI + job queue (test mode)
- app = APIPod(orchestrator="socaity", compute="serverless", provider="localhost")
+def test_simulate_serverless_runpod():
+ # serverless-runpod (no direct) -> Socaity emulation = FastAPI + job queue.
+ app = APIPod(simulate="serverless-runpod")
assert isinstance(app, SocaityFastAPIRouter)
assert app.job_queue is not None
-def test_socaity_serverless_runpod():
- # socaity | serverless | runpod -> RunPod router
- app = APIPod(orchestrator="socaity", compute="serverless", provider="runpod")
+def test_simulate_serverless_runpod_direct():
+ # serverless-runpod + direct -> RunPod local emulation.
+ app = APIPod(simulate="serverless-runpod", direct=True)
assert isinstance(app, SocaityRunpodRouter)
+ assert app.simulate is True
-def test_socaity_serverless_unsupported():
- # socaity | serverless | scaleway/azure -> Not supported
- for provider in ["scaleway", "azure"]:
- try:
- APIPod(orchestrator="socaity", compute="serverless", provider=provider)
- assert False
- except NotImplementedError as e:
- assert "not supported" in str(e) or "not implemented" in str(e)
-
-
-def test_local_dedicated():
- # local | dedicated | * -> FastAPI
- app = APIPod(orchestrator="local", compute="dedicated", provider="localhost")
- assert isinstance(app, SocaityFastAPIRouter)
- assert app.job_queue is None
-
-
-def test_local_serverless_localhost():
- # local | serverless | localhost -> FastAPI + job queue
- app = APIPod(orchestrator="local", compute="serverless", provider="localhost")
+def test_simulate_serverless_azure_warns_and_falls_back():
+ # azure has no serverless -> FastAPI + job queue (with warning).
+ app = APIPod(simulate="serverless-azure")
assert isinstance(app, SocaityFastAPIRouter)
assert app.job_queue is not None
-def test_local_serverless_runpod():
- # local | serverless | runpod -> RunPod router
- app = APIPod(orchestrator="local", compute="serverless", provider="runpod")
- assert isinstance(app, SocaityRunpodRouter)
-
-
-def test_local_serverless_unsupported():
- # local | serverless | scaleway/azure -> Not supported
- for provider in ["scaleway", "azure"]:
+def test_invalid_target_values():
+ for target in ("invalid", "serverless-invalid"):
try:
- APIPod(orchestrator="local", compute="serverless", provider=provider)
+ APIPod(simulate=target)
assert False
- except NotImplementedError as e:
- assert "not supported" in str(e) or "not implemented" in str(e)
-
-
-def test_invalid_enum_values():
- try:
- APIPod(orchestrator="invalid")
- assert False
- except ValueError as e:
- assert "Invalid ORCHESTRATOR" in str(e)
-
- try:
- APIPod(compute="invalid")
- assert False
- except ValueError as e:
- assert "Invalid COMPUTE" in str(e)
+ except ValueError as e:
+ assert "Invalid" in str(e)
if __name__ == "__main__":
- test_socaity_dedicated_auto()
- test_socaity_dedicated_localhost()
- test_socaity_dedicated_planned_celery()
- test_socaity_serverless_auto()
- test_socaity_serverless_localhost()
- test_socaity_serverless_runpod()
- test_socaity_serverless_unsupported()
- test_local_dedicated()
- test_local_serverless_localhost()
- test_local_serverless_runpod()
- test_local_serverless_unsupported()
- test_invalid_enum_values()
+ test_development_default()
+ test_simulate_serverless_default()
+ test_simulate_dedicated()
+ test_simulate_dedicated_azure()
+ test_simulate_serverless_runpod()
+ test_simulate_serverless_runpod_direct()
+ test_simulate_serverless_azure_warns_and_falls_back()
+ test_invalid_target_values()
+ print("All config resolution tests passed.")
diff --git a/test/test_core_functionalities.py b/test/test_core_functionalities.py
index 52ed88e..d645b58 100644
--- a/test/test_core_functionalities.py
+++ b/test/test_core_functionalities.py
@@ -9,7 +9,7 @@
from apipod import MediaFile, ImageFile, AudioFile, VideoFile, FileModel
from fastapi import UploadFile as fastapiUploadFile
-app = APIPod(compute="serverless")
+app = APIPod(simulate="serverless")
@app.post(path="/test_job_progress", queue_size=10)
diff --git a/test/test_streaming.py b/test/test_streaming.py
index f52d059..52e2d3d 100644
--- a/test/test_streaming.py
+++ b/test/test_streaming.py
@@ -1,7 +1,7 @@
"""
-Streaming over the serverless-localhost emulation.
+Streaming over the serverless emulation.
-APIPod on ``compute="serverless", provider="localhost"`` mimics a real
+APIPod on ``simulate="serverless"`` mimics a real
deployment: streaming endpoints are queued, the in-process worker produces their
chunks into a :class:`StreamStore` (the in-memory ``LocalStreamStore`` by
default), and the client consumes them from ``GET /stream/{job_id}``. A client
@@ -35,8 +35,8 @@
def build_client() -> TestClient:
- """Build a serverless-localhost app with streaming endpoints."""
- app = APIPod(orchestrator="local", compute="serverless", provider="localhost")
+ """Build a serverless-emulation app with streaming endpoints."""
+ app = APIPod(simulate="serverless")
@app.endpoint("/text")
def stream_text():
@@ -91,11 +91,11 @@ def _collect_sse_data(client: TestClient, stream_url: str) -> list[str]:
def test_default_stream_store_is_local():
- """Serverless localhost gets a LocalStreamStore by default; plain FastAPI does not."""
- serverless = APIPod(orchestrator="local", compute="serverless", provider="localhost")
+ """Serverless emulation gets a LocalStreamStore by default; plain FastAPI does not."""
+ serverless = APIPod(simulate="serverless")
assert isinstance(serverless.stream_store, LocalStreamStore)
- plain = APIPod(orchestrator="local", compute="dedicated", provider="localhost")
+ plain = APIPod(simulate="dedicated")
assert plain.stream_store is None
From 96797211364c31dbffd425e071c48a2aab6b444e Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Thu, 18 Jun 2026 05:33:32 +0200
Subject: [PATCH 15/38] .
---
docs/AGENTS.md | 83 +++++++++++++++++++++++++++++++
docs/EXAMPLE_DOCKERFILE_AZURE | 32 ------------
docs/EXAMPLE_DOCKERFILE_RUNPOD | 40 ---------------
test/test_core_functionalities.py | 11 ++--
4 files changed, 88 insertions(+), 78 deletions(-)
create mode 100644 docs/AGENTS.md
delete mode 100644 docs/EXAMPLE_DOCKERFILE_AZURE
delete mode 100644 docs/EXAMPLE_DOCKERFILE_RUNPOD
diff --git a/docs/AGENTS.md b/docs/AGENTS.md
new file mode 100644
index 0000000..9c1ebf8
--- /dev/null
+++ b/docs/AGENTS.md
@@ -0,0 +1,83 @@
+# Development instructions
+
+## 1. Think Before Coding
+
+**Don't assume. Don't hide confusion. Surface tradeoffs.**
+
+Before implementing:
+- State your assumptions explicitly. If uncertain, ask.
+- If multiple interpretations exist, present them. Don't pick silently.
+- If a simpler approach exists, say so. Push back when warranted.
+- If something is unclear, stop. Name what's confusing. Ask.
+- Use thinking methods like Elon Musk's first principle reasoning or design-thinking for complex tasks.
+
+## 2. Simplicity First
+
+**Minimum code that solves the problem. Nothing speculative.**
+- No features beyond what was asked.
+- No abstractions for single-use code.
+- No "flexibility" or "configurability" that wasn't requested.
+- No error handling for impossible scenarios.
+- If you write 200 lines and it could be 50, rewrite it.
+- Don't repeat yourself.
+
+Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify.
+
+## 3. Surgical Changes
+
+**Touch only what you must. Clean up only your own mess.**
+
+When editing existing code:
+- Don't "improve" adjacent code, comments, or formatting.
+- Match existing style, even if you'd do it differently.
+- If you notice unrelated dead code, mention it - don't delete it.
+
+When your changes create orphans:
+- Remove imports/variables/functions that YOUR changes made unused.
+- Don't remove pre-existing dead code unless asked.
+
+The test: Every changed line should trace directly to the user's request.
+
+## 4. Code quality and style
+- Code like a brilliant senior full-stack software engineer with deep-expertise in python.
+- Write high-quality, developer-friendly and clean-code, use design-patterns (Refactoring Guru) where helpful.
+- Include docstrings and comments to explain the why of a non-obvious method or code. No decorative comments.
+- Write memory and compute efficient code.
+- Write pythonic code and use builtins when it simplifies (generators, list-comprehensions, functools).
+- Reduce complexity and boilerplate. Remove unused code.
+- State unclear variable names and rename for better readability and clearity.
+- Avoid local imports
+- Do not write "deprecated" or backward compability code.
+- Update the README.md if there's user related important changes. Update TECHNICAL_README.md for architectural, or project maintainer important changes. Keep the .md update brief use technical correct terms.
+
+## 5. Reason and thinking style
+Do thinking and reasoning like a smart caveman.
+- Cut all filler, keep technical substance.
+- Drop articles (a, an, the), filler (just, really, basically, actually).
+- Drop pleasantries (sure, certainly, happy to).
+- No hedging. Fragments fine. Short synonyms.
+- Technical terms stay exact. Code blocks unchanged.
+- Pattern: [thing] [action] [reason]. [next step]
+
+## 6. Public facing texts: Readmes, prints, logs.
+- Write as technically brilliant European expert who gives you straight answers, backs claims with data, and genuinely wants you to succeed.
+- Ttone spectrum Formality 35% (lean casual) · Humor 25% (dry, never flippant) · Complexity 40% (lean technical) · Confidence 80% (very confident).
+- Direct without being cold. Say what you mean, no corporate padding. Warm without being informal. Complete sentences, measured enthusiasm. Committed, not hedging.
+- Committed, not hedging. "This works", not "this might work".
+- "We" for the platform, "you" for the reader. Reserve "I" for rare authored guides only.
+- Developer-first, technical correct terms.
+- No em-dashes. The character `—` (U+2014) must not appear in ANYWHERE. Use a period, comma, parentheses, or colon.
+- No marketing adverbs like seamlessly, effortlessly, robust, powerful, cutting-edge, automatically, intelligently, revolutionary, game-changing, world-class, best-in-class. E.g. three steps, ~X minutes" beats "easy onboarding".
+- No bullet-everything. Bullets only for true enumerations (statuses, regions, tiers).
+- Subtle transmit socaity's vision and use emotional-intelligence and marketing techniques for end-user intent based messages.
+- Use google docstring format.
+- The four brand hues (each tied to an ICP) with Tonal scales (50/300/500/700/900). Apply where makes sense.
+| Name | Hex | Meaning / ICP |
+| ---------------------------- | ----------- | ------------------------------------------------------------------------- |
+| **Neon Green** | `#5A8F00` | Developer. Active system health, engineering precision, technical truth. |
+| **Silicon Blue** | `#0A86BF` | SMB CTO. Enterprise stability, secure infrastructure, professional trust. |
+| **Creator Violet** | `#7C3AED` | Creator. Generative potential, creative momentum, warmth. |
+| **Signal Pink (Rose)** | `#EC4899` | Alert. Warnings, critical status, high-priority attention. |
+
+# About Socaity
+
diff --git a/docs/EXAMPLE_DOCKERFILE_AZURE b/docs/EXAMPLE_DOCKERFILE_AZURE
deleted file mode 100644
index d9d8aac..0000000
--- a/docs/EXAMPLE_DOCKERFILE_AZURE
+++ /dev/null
@@ -1,32 +0,0 @@
-FROM python:3.10-slim
-
-# Set the working directory
-WORKDIR /app
-
-# ffmpeg needed for media-toolkit video
-# Install FFmpeg and dependencies required to install ffmpeg
-RUN apt-get update && apt-get install -y \
- software-properties-common \
- apt-transport-https \
- ca-certificates \
- ffmpeg \
- && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Copy the shared connectors module
-COPY . /app
-
-# Install dependencies
-RUN pip install --upgrade pip
-RUN pip install .
-
-# Configure environment variables
-ENV HOST = "0.0.0.0"
-
-# Expose the port
-ARG port=8000
-ENV PORT=$port
-EXPOSE $port
-
-
-# Set the entrypoint with explicit host and port binding
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
diff --git a/docs/EXAMPLE_DOCKERFILE_RUNPOD b/docs/EXAMPLE_DOCKERFILE_RUNPOD
deleted file mode 100644
index c302cd0..0000000
--- a/docs/EXAMPLE_DOCKERFILE_RUNPOD
+++ /dev/null
@@ -1,40 +0,0 @@
-FROM runpod/pytorch:2.4.0-py3.11-cuda12.4.1-devel-ubuntu22.04
-
-SHELL ["/bin/bash", "-c"]
-
-# Install dependencies
-# ffmpeg needed for mediatoolkit video
-# Install system dependencies including CUDA and cuDNN
-RUN apt-get update && apt-get install -y \
- ffmpeg \
- libcudnn9-cuda-12 \
- libcudnn9-dev-cuda-12 \
- && apt-get clean
-
-# Ensure CUDA and cuDNN libraries are in the library path
-ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}"
-
-# create the folder for face2face
-WORKDIR /
-
-# Install Python dependencies (Worker Template)
-# added runpod directly in dependencies
-# added other libs before which keep failing in requirements file for unknown reasons.
-# note that fairseq need to be installed differently in windows and linux
-RUN pip install runpod>=1.7.7 && \
- pip install . --no-cache
-
-# Configure APIPod for RunPod serverless deployment
-
-ENV APIPOD_COMPUTE="serverless"
-ENV APIPOD_PROVIDER="runpod"
-
-ARG port=8080
-ENV APIPOD_PORT=$port
-# allows the docker container to use the port
-EXPOSE $port
-# allows any IP from the computer to connect to the host
-ENV APIPOD_HOST="0.0.0.0"
-
-# Set the entrypoint with explicit host and port binding
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
\ No newline at end of file
diff --git a/test/test_core_functionalities.py b/test/test_core_functionalities.py
index d645b58..01bf5c8 100644
--- a/test/test_core_functionalities.py
+++ b/test/test_core_functionalities.py
@@ -56,13 +56,12 @@ def test_single_file_upload(
):
return file1.to_base64()
+@app.endpoint("/normal_endpoint")
+def normal_endpoint():
+ return "normal_endpoint"
+
+
-@app.endpoint("/make_fries", methods="POST")
-def test(
- mymom: str,
- file1: fastapiUploadFile
-):
- return "nok"
if __name__ == "__main__":
From 1ab3e6862dd63cc031cd354ce518a2c7c2d07509 Mon Sep 17 00:00:00 2001
From: w4hns1nn
Date: Thu, 18 Jun 2026 08:15:17 +0200
Subject: [PATCH 16/38] set-up a comprehensive test-suite for all apipod
features including different build configs, endpoints, media support and
streaming
---
.github/workflows/publish.yml | 34 +-
README.md | 2 +-
docs/AGENTS.md | 12 +-
docs/TECHNICAL_README.md | 24 ++
pyproject.toml | 29 ++
test/__init__.py | 0
test/conftest.py | 141 +++++++++
test/files/hermine_clone.wav | Bin 0 -> 1115244 bytes
test/files/socaity_logo_for_white_bg.png | Bin 0 -> 335078 bytes
test/files/test_video_ultra_short_short.mp4 | Bin 0 -> 186424 bytes
test/llm/test_llm_mock.py | 173 ----------
test/llm/test_qwen_llm.py | 333 --------------------
test/services/__init__.py | 11 +
test/services/core_service.py | 103 ++++++
test/services/schema_service.py | 117 +++++++
test/services/streaming_service.py | 33 ++
test/test_cli.py | 89 ++++++
test/test_clients_six_schemas.py | 185 -----------
test/test_config.py | 79 +++++
test/test_config_resolution.py | 74 -----
test/test_core.py | 84 +++++
test/test_core_functionalities.py | 73 -----
test/test_demo_fastapi.py | 48 ---
test/test_schemas.py | 101 ++++++
test/test_service_six_schemas.py | 62 ----
test/test_streaming.py | 201 +++---------
26 files changed, 899 insertions(+), 1109 deletions(-)
delete mode 100644 test/__init__.py
create mode 100644 test/conftest.py
create mode 100644 test/files/hermine_clone.wav
create mode 100644 test/files/socaity_logo_for_white_bg.png
create mode 100644 test/files/test_video_ultra_short_short.mp4
delete mode 100644 test/llm/test_llm_mock.py
delete mode 100644 test/llm/test_qwen_llm.py
create mode 100644 test/services/__init__.py
create mode 100644 test/services/core_service.py
create mode 100644 test/services/schema_service.py
create mode 100644 test/services/streaming_service.py
create mode 100644 test/test_cli.py
delete mode 100644 test/test_clients_six_schemas.py
create mode 100644 test/test_config.py
delete mode 100644 test/test_config_resolution.py
create mode 100644 test/test_core.py
delete mode 100644 test/test_core_functionalities.py
delete mode 100644 test/test_demo_fastapi.py
create mode 100644 test/test_schemas.py
delete mode 100644 test/test_service_six_schemas.py
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 54d963d..f8df7e2 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -1,11 +1,41 @@
-name: Publish to PyPI
+name: CI & Publish
on:
push:
branches: [main]
+ pull_request:
jobs:
+ # Tests gate everything: no version bump, no publish unless these pass.
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Install package + test deps (incl. fastSDK)
+ run: pip install -e ".[test]"
+
+ - name: Run tests
+ run: pytest -q
+
+ # Soft checks: report only, never fail the build.
+ - name: ruff (soft)
+ continue-on-error: true
+ run: ruff check apipod test
+
+ - name: mypy (soft)
+ continue-on-error: true
+ run: mypy apipod
+
+ # Runs only on push to main, and only after tests pass.
publish:
+ needs: test
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -18,7 +48,7 @@ jobs:
with:
python-version: "3.12"
- - name: Install dependencies
+ - name: Install build tooling
run: pip install build twine
- name: Bump patch version
diff --git a/README.md b/README.md
index 929ed03..a2b3c12 100644
--- a/README.md
+++ b/README.md
@@ -60,7 +60,7 @@ def hello(name: str):
return f"Hello {name}!"
# Built-in media processing — uploads/URLs/base64 are parsed for you
-@app.endpoint("/process_image")
+@app.endpoint("/process-image")
def process_image(image: ImageFile):
img_array = image.to_np_array()
# ... run your AI model here ...
diff --git a/docs/AGENTS.md b/docs/AGENTS.md
index 9c1ebf8..03f74f6 100644
--- a/docs/AGENTS.md
+++ b/docs/AGENTS.md
@@ -39,7 +39,6 @@ When your changes create orphans:
The test: Every changed line should trace directly to the user's request.
## 4. Code quality and style
-- Code like a brilliant senior full-stack software engineer with deep-expertise in python.
- Write high-quality, developer-friendly and clean-code, use design-patterns (Refactoring Guru) where helpful.
- Include docstrings and comments to explain the why of a non-obvious method or code. No decorative comments.
- Write memory and compute efficient code.
@@ -49,7 +48,13 @@ The test: Every changed line should trace directly to the user's request.
- Avoid local imports
- Do not write "deprecated" or backward compability code.
- Update the README.md if there's user related important changes. Update TECHNICAL_README.md for architectural, or project maintainer important changes. Keep the .md update brief use technical correct terms.
-
+- Write flake8 formatted code with "flake8.args": [
+ "--max-line-length=120",
+ "--ignore=E203,W503,E501,E261,W293"
+ ]
+}
+- Avoid using direct cmd python calls and pip installs. Work with the local venv if necessary. Always ask if necessary.
+
## 5. Reason and thinking style
Do thinking and reasoning like a smart caveman.
- Cut all filler, keep technical substance.
@@ -78,6 +83,5 @@ Do thinking and reasoning like a smart caveman.
| **Silicon Blue** | `#0A86BF` | SMB CTO. Enterprise stability, secure infrastructure, professional trust. |
| **Creator Violet** | `#7C3AED` | Creator. Generative potential, creative momentum, warmth. |
| **Signal Pink (Rose)** | `#EC4899` | Alert. Warnings, critical status, high-priority attention. |
-
-# About Socaity
+
diff --git a/docs/TECHNICAL_README.md b/docs/TECHNICAL_README.md
index 24dd744..2a9473f 100644
--- a/docs/TECHNICAL_README.md
+++ b/docs/TECHNICAL_README.md
@@ -144,4 +144,28 @@ The **stream store** is the pluggable backend that buffers a job's chunks betwee
A client calls `POST /tts` with a JSON body. FastAPI validates it against the rewritten signature and calls the outermost wrapper. The file-handling layer converts any media inputs into media-toolkit objects. If the endpoint is queued, the queue mixin stores the job and returns `{job_id, status: "queued", links}` — the worker thread later executes the real function, feeding `JobProgress` updates into the store. The client (typically fastSDK) polls `/status/{job_id}` until `finished` and receives the result, with any returned `AudioFile` serialized as a `FileModel`. Without a queue the same conversion happens inline and the response returns directly. On RunPod, the identical user function is reached through `handler → _router(path) → file handling → execute`, proving the core principle: the function is written once, the backend decides how it runs.
+## Testing and CI
+
+The suite (`test/`) is built so endpoint definitions live apart from test logic, and one helper boots them under any run intent.
+
+```
+test/
+├── conftest.py # build_service (in-process TestClient), live_service (real subprocess via CLI), config matrix, file paths
+├── services/ # reusable services: register(app) callbacks + a runnable entrypoint
+│ ├── core_service.py # scalars, custom model, mixed media, JobProgress, file upload
+│ ├── schema_service.py # one endpoint per standardized schema, an extended schema, raw/typed mapping CASES
+│ └── exec_service.py # runnable service for fastSDK + streaming (predict, echo_image, chat, text, video)
+├── files/ # media assets for upload/download tests
+├── test_config.py # intent -> backend resolution + /openapi.json loads for every backend
+├── test_core.py # core endpoint plumbing (types, model, files, queue lifecycle)
+├── test_schemas.py # standardized schema endpoints + response-model normalization
+├── test_cli.py # scan / build / simulate produce the right artifacts
+└── test_execution.py # fastSDK end-to-end (subprocess) + SSE streaming (in-process)
+```
+
+Run `pytest`. Two primitives keep tests DRY: `build_service(register, **config)` yields a `TestClient` for an app built from a `register(app)` callback under any `APIPod(**config)` intent; `live_service(simulate=...)` boots a service as a real subprocess through `apipod --start` / `--simulate` and yields its URL. Backends are parametrized from `FASTAPI_CONFIGS`, so a single test asserts behaviour across development, serverless, dedicated and runpod.
+
+The fastSDK end-to-end tests skip automatically when the installed fastsdk lacks the `connect` API (e.g. a local in-progress build); CI installs one that has it. Pytest config (`pythonpath` in `pyproject.toml`) resolves `import apipod` to the in-repo source and the `conftest`/`services` helpers by name, so no `sys.path` shims are needed and each file is also runnable directly from an IDE via its `__main__` block.
+
+CI (`.github/workflows/publish.yml`) runs the `test` job on every push and pull request (installing `.[test]`, which includes fastSDK). The `publish` job `needs: test`, so the patch version bump and PyPI upload happen only on a push to `main` after tests pass. flake8 and mypy run as soft checks (`continue-on-error`): they surface warnings without gating the build.
diff --git a/pyproject.toml b/pyproject.toml
index 4561b03..f7c9f54 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,8 +28,37 @@ apipod = "apipod.cli:main"
runpod = [
"runpod>=1.7.7"
]
+test = [
+ "pytest",
+ "httpx",
+ "ruff",
+ "mypy",
+]
[project.urls]
Repository = "https://github.com/SocAIty/apipod"
Homepage = "https://www.socaity.ai"
+
+[tool.pytest.ini_options]
+testpaths = ["test"]
+# Make `import apipod` resolve to the in-repo source (not an installed copy) and
+# let test modules import the shared `conftest` helpers and `services` package by
+# name. The sibling fastsdk entry is used in local dev when the repo is checked
+# out next to apipod; it is silently ignored in CI where fastsdk comes from pip.
+pythonpath = [".", "test", "../fastsdk"]
+
+[tool.ruff]
+line-length = 120
+exclude = ["venv", "dist", "build", "*.egg-info"]
+
+[tool.ruff.lint]
+# Mirror the previous flake8 config: ignore stylistic rules that conflict with
+# black/our formatting conventions.
+ignore = ["E203", "E261", "E501", "W293"]
+
+[tool.mypy]
+# Soft static typing: a warning signal, never a build gate (see CI).
+ignore_missing_imports = true
+follow_imports = "silent"
+warn_unused_ignores = false
diff --git a/test/__init__.py b/test/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/test/conftest.py b/test/conftest.py
new file mode 100644
index 0000000..74b5f7c
--- /dev/null
+++ b/test/conftest.py
@@ -0,0 +1,141 @@
+"""
+Shared test infrastructure for the APIPod suite.
+
+The suite is built on a few small primitives so individual test files stay
+declarative and DRY:
+
+- ``FASTAPI_CONFIGS`` / ``QUEUE_CONFIGS`` — the canonical matrix of APIPod run
+ intents (development, serverless, dedicated, runpod, ...). Parametrize over
+ it to assert behaviour holds across every backend.
+- ``build_service(register, **config)`` — build an app from a callback that
+ registers endpoints and hand back a wired FastAPI ``TestClient`` (in-process,
+ no socket). Used by the config / openapi / core / schema tests.
+- ``live_service(simulate=...)`` — boot the example service in a subprocess via
+ the real ``apipod`` CLI (the same ``--start`` / ``--simulate`` a user runs),
+ yield its base URL, and shut it down afterwards. Used by the fastSDK
+ end-to-end execution tests.
+
+A registrar is just ``def register(app): ...``; keeping endpoints out of the
+helpers lets one helper serve config, openapi, schema and execution tests alike.
+"""
+
+import contextlib
+import socket
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+from apipod import APIPod
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+# --------------------------------------------------------------------------- #
+# Test asset files (used by the media upload/download tests)
+# --------------------------------------------------------------------------- #
+FILES_DIR = Path(__file__).parent / "files"
+IMAGE_FILE = FILES_DIR / "socaity_logo_for_white_bg.png"
+AUDIO_FILE = FILES_DIR / "hermine_clone.wav"
+VIDEO_FILE = FILES_DIR / "test_video_ultra_short_short.mp4"
+
+# Entrypoint booted by the fastSDK end-to-end execution tests.
+INFERENCE_SERVICE = Path(__file__).parent / "services" / "core_service.py"
+
+
+# --------------------------------------------------------------------------- #
+# Run-intent matrix. Each entry is the kwargs passed to APIPod(...).
+# --------------------------------------------------------------------------- #
+# FastAPI-backed intents resolve to SocaityFastAPIRouter and expose a real HTTP
+# app (the only ones that can serve /openapi.json or run under TestClient).
+FASTAPI_CONFIGS = [
+ pytest.param({}, id="development"),
+ pytest.param({"simulate": "serverless"}, id="serverless"),
+ pytest.param({"simulate": "dedicated"}, id="dedicated"),
+ pytest.param({"simulate": "dedicated-azure"}, id="dedicated-azure"),
+ pytest.param({"simulate": "serverless-runpod"}, id="serverless-runpod"),
+ pytest.param({"simulate": "serverless-azure"}, id="serverless-azure-fallback"),
+]
+
+# Intents that boot a job queue (serverless emulation): endpoints return a
+# JobResult processed by the in-process worker.
+QUEUE_CONFIGS = [
+ pytest.param({"simulate": "serverless"}, id="serverless"),
+ pytest.param({"simulate": "serverless-runpod"}, id="serverless-runpod"),
+]
+
+
+def build_app(register, **config):
+ """Create an APIPod app for ``config`` and register endpoints via callback."""
+ app = APIPod(**config)
+ register(app)
+ return app
+
+
+@contextlib.contextmanager
+def build_service(register, **config):
+ """
+ Yield a ``TestClient`` for an app built from ``register`` under ``config``.
+
+ The client is entered as a context manager so the app lifespan runs: that
+ starts the in-process queue worker for serverless intents.
+ """
+ app = build_app(register, **config)
+ fastapi_app = app.app
+ fastapi_app.include_router(app)
+ with TestClient(fastapi_app) as client:
+ yield client
+
+
+# --------------------------------------------------------------------------- #
+# Live server (real subprocess via the apipod CLI) for fastSDK tests
+# --------------------------------------------------------------------------- #
+def _free_port() -> int:
+ with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _wait_until_healthy(url: str, proc: subprocess.Popen, timeout: float = 40.0):
+ """Poll GET /health until the service answers, or fail if the process dies."""
+ deadline = time.time() + timeout
+ while time.time() < deadline:
+ if proc.poll() is not None:
+ raise RuntimeError(f"Service process exited early with code {proc.returncode}.")
+ try:
+ with urllib.request.urlopen(f"{url}/health", timeout=2) as resp:
+ if resp.status == 200:
+ return
+ except (urllib.error.URLError, ConnectionError, OSError):
+ time.sleep(0.2)
+ raise TimeoutError(f"Service at {url} did not become healthy within {timeout}s.")
+
+
+@contextlib.contextmanager
+def live_service(simulate: str = "serverless", entrypoint: Path = INFERENCE_SERVICE, port: int = 0):
+ """
+ Boot ``entrypoint`` in a subprocess through the real apipod CLI and yield its URL.
+
+ ``simulate=None`` runs ``apipod --start`` (development); otherwise it runs
+ ``apipod --simulate `` exactly as a user would on the command line.
+ """
+ port = port or _free_port()
+ cmd = [sys.executable, "-m", "apipod.cli"]
+ cmd += ["--start"] if simulate is None else ["--simulate", simulate]
+ cmd += ["--entrypoint", str(entrypoint), "--host", "127.0.0.1", "--port", str(port)]
+
+ proc = subprocess.Popen(cmd, cwd=str(REPO_ROOT))
+ url = f"http://127.0.0.1:{port}"
+ try:
+ _wait_until_healthy(url, proc)
+ yield url
+ finally:
+ proc.terminate()
+ try:
+ proc.wait(timeout=10)
+ except subprocess.TimeoutExpired:
+ proc.kill()
diff --git a/test/files/hermine_clone.wav b/test/files/hermine_clone.wav
new file mode 100644
index 0000000000000000000000000000000000000000..fd5fe0c4a9589c4923d379464b06e702f9d7aab3
GIT binary patch
literal 1115244
zcmW(-1ALt8(|x_RlC(+N6iIE{ws~vYwr!`qwQbzmPLU#QsyOT2*ZSuF{q`5`HaD5)
znVB=^oVi_Ew`ehxupvDg_h>$R+|;601VK=EG}?;#@lBvUz56U}^W
zwtl%O)sktG9j9Ef>%Hr=`@KiUS%Y4DK%y`3)r^oc8b#b~B?H$cKsxoz(Y(kpJ@njY$lX|KK)t+ibX{jmXDB=~C
zjZ{`I%NwM4skGEb`Yx@MAIqoZ*K%FuKc%v|OifpdA|Z9Ony7wO82OK+k>W(V(2>{i
z4!#{`!AZvJb
zot##10%>_l{uu9ApC<4=e4HI5R#PgX!|+-7c&s7DW7}{J|3RcsgjS<%rFH158f%!p
zSjRhJT-Dq^+@iaq=Zoi!r>keA`=e8`zqD>MS2xWxjyHBU(WX?>8B><2z4^FVFl#Kw
z%`MD#OwWy14eRxjb?3EuZ6VED@;XtPIF66NAL5OP|A-P~b8;_Pi>gcYA`21eSPyi*
z`d+>-m61M)vbajRC#{xJB)hy;F02HU&+1C#1#%yGhipWqsx_2Z@)oJ4$nz|Rb1T>@
ztclyjE#(IaPVu?ePTDIylRij)rB~8a$snB;CyQmo>%w~Bg78-0gnnYI#7PI`wn|xu
zcVDCxQX6RoXYf|tq?S;hE4P#^B~h)f)`GW|YHPKV+DKKEN6K0yUb!uwkmt)9c?H~-
zt>Rn3%Uk){TvN6TQ-N+3O^Ey#>K`l-_~{$zEmGjg59j9O#N~9&K9p4<3xT)KS)Z~u
z=e&cnSd^EUU&h*A13epJakF-T=U{A0e_y}SV@rpP{4b=@dj5Q6h
zrrX~-COGT49=oQwlRY`^2d;mdZ|u0Onq{5owQ-86i1~}@m`OF&Hm`&e=xe!WsbNK}
zdP~4m#dO-JF(w;2>7VK<>WXO%np)I<FBNFR~pumWaVup@ont
z>QCjBvS0Z}DX!?0I7JVS9ZExWrFv0)2RHSAI#BJXc2TRUgt|j{E&r8l(gLxhm?fkM
zMzO6pR9q|m5Q|C^rCg~Z_c8R+A$<*V$iT`)QSVxA>$R_?c){W&Lcj^?1bcXE5?
zoy%kLR^=}(xaw`{ZxNgweiN<8j^(?GIZ|WAtTs`nswna=5=I80JFuC=HtN1MNAERW
zG&i(9w1w@X9X%aY9pxPs#~1r*J7J$@t7hY^Z>@|q&wAE+$NJ2AziH)MJ^zxkxocZJqtCelX6y$m*>HW
zoQ0F9C{_@wiJiq>;uP_<=#(syC>EBQNgbup(q_n#7`cNyOMW7om0rqyr3%FJp~|b}
zkzU9=ACas
zisc{8A6;<6JH|gQ_%dvxdFDPhP&gsxNl|%)Qb(&*@4rgPi4OWaZJAor6q^ESRXUx)9*Z{#oWpLkR#A@mn^2#*Av*hf4l=8Fxb3zAtb
zDR+TB)LfYik6}uErG!#fiBZgX7a9-FV)Dy<=Be}k;mDwD%
zM!JXk1=jg=-jn$=^0wp}a`)#n%Sp;nvkzzY&aRl9nB6-2diKbigxo-`47F%TL3V-J
zhXfpB_(VmDSVu-c)9?XaKLcb%o
zkhMrrib>3
z#|0_*X?gwgjCp5rE94%`>5vnjb1QpX_PT5aA0I;M2p+e?s6JjJ6%6rcU>b~HC(k_znvxE`_rAN&dE-vvw?H4lZBI*4w=);w!(T7@}R%D
zG@ORnG}f44bQ{kbe(00*hjhhs)3qlwQEC8{Ms_8S5P$K`_%kdP8;<5dMA{)=)jjG$
z=#dF3qa0G|C@|yped!0SY
zo@bxJ+fkNbTf*6#;;j5(zOpc0I4krNtTR{U`b+txTvx^@O_cHquH?uW@_xCrd$ve0X%&6#rFdrNo!2GB$$vdu~
zNP)fJTD~d&T;A!t`FW+GFU-hGf(n#a5Gi==o#(6MuMl_~42PZcD`p>;CkRq~rHQ&7
z`Heosj^L|_vZR)}sd=G0Y*=mbTQ=GUJ44RauA=S}?rHAw?tp8L>xe7P9p~QUdf>`*
zuk%#*JaON1e{}D1Tiws#OkTOp!ZXKR16?g#F4rtLH_VZ2ciRls?v`xx2y?1wxT%;a
zY782m7%LjP8s_RB=+d=UHGip*)Mv6jd7tP*q~qoAjhGV~jXpx=Al(3;PE$*&zv0d%
z!5NH_MQNCHO3V~Od?pvqjbrCA@9C5DPI?hNgFZ@EV|FqBGN+gq%v0t%bCuc2^k5n@
zQ{i)jO=3&3ec7LEPwpyLlE1-E5KQ7UkryvY$#PfulH5pHp{!OmDf^Vo$_%BQ5~~;$
zv+`5kASVKbZVJ7{EbRj9nHflg|W`E&v!I}ZrsRm#j(vX!ZF^l)1il4;G9Ju2S&Jh
zxH$M~x$}+F;`;2o?2K{sffLy1j6jvT1HGh~t%r4kCC;+aeA6TuM?xl;jVBCVeJ6be
zRHfV6Pnuk+3N;$?z)B2-j;zOe0TP*lkjNo*m|6|c&}K!D56F|`TJjS>o`c1ELMohq
zgDcGrVSdy7=ZJ(F@TpQ7&4QUQXYoGiW_ik7>k|VGhiwUp{sDb=rJ!Xsb#q1=Wp{#@Csj8!@*
z9_6y!T{g&jp(b7vw(uRf?aa9-5$OdrDb^qIdJ8^67TnDnomY^%Ft>W{U#LV6A@&`i
zC*RKNlD|E_NI_9=bzdp}y1Ep1z&E7W(?aNKZsoN11Oj<1eZ5NkWsqd#!(1FmI|Cs*Jsw!&8jT=U@j4_r%K
z7S|i7O6MH3y`+7N?WeVc^*>7)%Xl+q`fW-vQKsoe)VS8b>uc*<>H28*YP?iAY9x6L
zy255W0Y3=2uoiWo$Dk(ZkW_Vu`d!(jbOCHyPhKP)75zf8P!*8L6!t5#iK)w^LF{ws
zA`tmL%vRyj3Y*15m?-wajMyD!
zM4zmMemq&ZpnO$+D~gf~8OSMV$^*!b!H^j_fCie&gJe{mFV&F#i1owO
z1g14Ic+P*=x81w6AT@t%zAvv&9+!I!qMw$tHfI^2)M(D-+`D;w^E2|Z3u1ij{l5dh
zLs;}W9b`6hSAx2JG{3O?CF6!PGKALhYFKl(76ZoC8VS29VZs}g&
ze&YV@{^n-g)jgLzonv;!OpU1@6C0E6>Fr7IwDwH*9P_;Q{PaBYjPfY%4ep<=Qm*Ar
z8M^pMd&sum*1`70+TZ%w(%tgPYzMs5&v?oZ4|VCQZm2F-TUUEnQ&)4Jil>&69`Ytp
zk2sI_#3igN_7Ko&3G@h(jI4n^*-&{TH-HX(LF9yn!fO5-*N>~t8MycC3-%85fGg}X
z_7nR99tbg`%OMnYvLOx?@nJ@S9@
zH#ti-C^eLdiYh;cIc=vr9+2=$X|7aMx+r!MJ>nN(gpkfB@MXC|>}PsrG#oA(nj2{2
zxA`8x8MMv6ooCNmm|HCOc~0LPJSUv}Gy6(5ll?GfRPKe`(RtVMZx!@~{xCMMD%3CH
zrgNCbTrXjv_*$x?)Km#18C{Cq!+#NzsEb-gf6h47yuuo_4Y8-%w>eBs!Qpdsccwc@
zS4Ft-ysNT%v%92ch{xu+3Y}wtn{t1J9@5R-!Cli`)E$8ylIWW2{N)(x2-|zu=h&`W
z+gW3*k1TC1AI$a5>r5F&+$ic3^^0_mwC^-usEX7RvK-lkIDuDy%2XZuh|Wfn(Pu~p
zNo4RH|wc%lkp5`72eVK*K
zxa8grpI3(N-PxT4w?E>l<8JKEgC0HFmEmj!RS5%(E5K~wwQjOju|9=NV9Ybjxuz+m
zC&s8D$uL5HO&8Rb)y~tTQDdkNp(?ptWFP&>>sZY(Oaopf^{5X*ow|
zA)Mxm@@F9`MIkEZ0S&!_Ea=a+V>__J*x7J4Mc68=nSIZ^WqvYwj2WI;#eQcya+f)k
zo5+iNf8n5zCsY-e0PB${=8A+=SLzSgXR0(uY6%l@F)2~X6CaAV#cg71z{MYh<&b$H
zeiom@HRBGmc}#>>qg3>MxL3#(%=f4GCVQ_GxC#d6f6ZHy*B91T>J&aITWqdA?NA}a`b$@NO5+Bd%eeX#T5_tyc$INiQDBV>8b4*=SlHw1#SuTymtQvexrq_ipK>k
z#|t=_qVBCO4C+ymbC9E-{g>^7Z86MxMXYO~KYTDZGA}p1G1`m+42$)vbZOe!+DV!l
zl#7~1@bDk@#Tvc8xJrlL!0O1l}hQG&6<63ei?i>4%
zT?f-bJBa)z<`N*J?aVCb5G5cxvgwcXMfx25mR9Iu(53e<1~{Y7YCcIb?!4
zk_;$yLBIkNb2cE<2KmMFCgx7e9g>?2XbFY>d^lIiZIpKVl42e1U+GY%*>cU@NsHPpP2G7S)Tiz3!a6ZCZ0kbgXaYx+958j>xHwe
zbEhLbnmH6rb|<$HG(H*U
zv8mW!bU3Oa`(TRltMgQw`UrYuro2*a2=nX$$c%JxuQ*j~AqIuHLIa_OU=Tj>S0T2|
zU{00#ZXqD*d$hz?2(3PMCW`_O#
zPrex625-FgctQVyk_EZ>AM@|!ugYJZpP4_ez+0g64)Rv<754l5nUD{*FcB#mT~0q>
zez0ZvGXgKtQj&5*)uO*p6+2Fts37IhKG3x^3^EQjO)}52G_*dpF0eW6PP+}*j9j~9
z58F#SHakAREceP$*E!ai;6$7qolBfuV1jGqoaGz_Uv-4<-hl2r!cobw8~*JJTQA!c
zYd!04%QTC|vd3H%*wB?3ZKJ2-=F4h5;zm+6C51s7j7M?5%tjN^e$!}o6A)b
z%86ldl^j(|qLYw5NCPaMs7S7)#sb6eO4CXAOz+i~)_>E}#+SgzC79Nj%UR!BuUmoP
z1~y~4y$hVo0eiZ=hNH0KzI`ijN=Ap#aoK*&UdmD2F$br$#$JYJ`q_F}w^7#$^5P%uU(GSiB+W0X7iFaGkz+`LyhrROS`*p$
zay$wDjSax=qb<=J$arJ`;zB;EC)6owUG*jK3?AhMutL|Qc~T3>Bjt)0#Es%?aiCa9
zd|S;$RGg7;-(@DkWWdaKlvxf{EEX!;NtjDY
z(7&VWqJyK|qIII0=IN&iXfQ=8TyOEJ-c`S-;
z!YdF#;tIKp>Zd88&CqVtHPlD-hYj6~6sS5Grt#)tmRXjQmIBChY8z5`ig?qz!TnL*Z{UVPdmEi1R>G$+#sAV!U
zk1YzwtQ22?9|Z`Z1t8@e(kQu!(njrsR7MRLi{;_Zh!tcP$_@9hsTR>4($&5QCW1U~SU0XxTX;L%}
z&1TTzNSGZoe!F$Y4ZP_o@+?EILOJG8eYhP&}Y0qeH
zYHw@%YiZ3d%~Q>M%}~uc%~j1g%^J-t%@mDWvyNIw9e^p|0=b`DN>(7B5qpSTL>t17
zhwvPHE8YNCv9H*9V6Z!Y&Z5EYp-<2>bQR2J70^oPTqFay&UDqK4pj;RpWgr!m5EX<
z=*MC4oVZ>zid_Um$lyl+il4%j<^HgH*ru!jy5|9=1K=Zs`AYBl|LLJQJ(pfXkESQl
zTj=ZbZD5=h(2tH@Waszarc8^}&%aqt^9k1pnSCuXQ
zKe1L;wEzG3BVUw{$-kAJM~b_Rl{lvWDlrCXOIk}6zCXD
z&}HaNR0q>yBWwzmg5AQtV+vLj)Fv%n1lQqZ;kNFm&kysn7JysJd3wmB#tS;6atB6I>59k~86M76?
zi7rArqlplSmB=Jy2r?b=WDT%r8;}D?3TV728Vhl9qM67e0V|vV>eXuCK^ws>4k%tFNBOH{fNpz1S*Z+xid{l6
zfL2pU=?zLsqS76dn*^mO=r(!seR(f%1yyCOoDP`hhICXq1R6lPWC8wcBRm_Dla%U;
zRrvw@=W&SeW{CJ!IR&2iAq#RD_}Pm=3;U$_f&KZaa0;T9RfntV;5RN%5u`Gx%k^On
zUw|Bf$iGJ-hz4zd&Onc%SJB&`^OwYGVgF)T7>U;hb$lJZ89#_;;D7P2cq)9h5buH)
z$1CG4@V@wTc=i&03%`ip#|z+0Vu>*R0Zw5aD1<2f4LgOc!6sszAiv9E9QqjDgib))
zqH$;eau;H}4w-;-hu@?<(
z{;O_RC#c=khN@2errZM+d5*FMxZ~q+^KSxMc~?0F888cwO%3?j+457k|7U?q?kl&G
z%gF)hopcpk0E?uN;0Cx3H{K-=k+;daKx>%`Z(Zav!2j4}QjSV8WXwZQG=GBb{{}w$
zB;`pO*$tUQ$R&V>8wXB;%fJv(N;3Snn!uTLP+BQ%fx&AIxc9K~2h^Uu>Sxskw`x9e
z0{M>ofGkNzZo`@Vi)@AX-9nxs50Ga_8oXVFSL{VrA)S##m?~osJ)%L(;L=Dy+`tC^
z1f6_}T3ihQR~u9W#R&JIJLn|G)C|>%;K*O~C*=PIP>?3U_dBbt)COvMaF}dR#G
zqv|R3jJg9l$XLkR{(v`zsl7pI>ID^Jh&oE$r#@HTs((}+VnkHcfmB240xqeIw1%0g
zALP^qWE_S#U)^$ZH!tWaoxibJ>e+B-M
zTj~O+l=cb>=woeXT)978!=nBD(;2}^O+Z9G|V6Q#E)c^}Lja)+a~eebKwB
z0r>~JjM!|jWfEOn(6MgXGpcQO_I(=FD$V3@#
zqGJ(8n_?VAWgtXpwT_jRnWgj3Gdb2hT4Yxio%4%IBK_cn=&Yx#RVzUumL&F?B*V0wR9uNMbcbW
z5;kKW$Ua0HMBobOo$6b?j~Xg!gjMkOn6b0!0(^j`8vaXg@F{9h%~d_2xi8h?>tWxu
zg9$_t<$CxLO;zF{-;bW8ywEk(uR(q?OX)4rchxM9jliq^Iw%BoYP_d<&3MrKnIbw!@WOcdr5L-b{cPr?T&h^@eJ?7rGpaiVpI
zpLi-_m&Z#L(L2O2WVUEix&WVB9=l5(*Se?~s9jy6R>YT456KDGUwNN!TKa)yQOyWD
zP&0h8UX-d}qW~YKpylNT
z%ouho)gHRujqbzDZ%>P;3W#tK3
z1V1X4h)|eB0CpKc-^b?^Of$3O$fiN^0M-P!dXtK4O+$CC){e{dXq6#Av
z295CmSCUIm%2Dl!^6cZ_EIya)r+uSD8MD{~8;ng>#zNm7FFTOZXjkQeWJd;%sRQ@|74&mB&WOMdTjRH{l_gY@DwfAkK*h{A%i$(WslOW<=XWiXk=3BaH_6
zYRDHwk;|In+A??-es%aOeF=MH+@e_{Zen)}1C#`{htx~jf?)U`w1qN@pUBlyG{j%*
ztCAomtEYfrm?7S#`!Hjq?+B~BR*GU6(v|BGX}~TT*Yhh-{d*gec#^`xdQ@T#N94bqlqTRKtJVvDiw_(bA8(p5SkiP(AlbW>05U}Tou
zLVbpICg1Av_5F!LVkhAiI#e@HGZ{mL&-6wyLaf*J)EuDt;RD4nY)QEextHpQ++bTq
z2k~hrOC@MiC_mOsZpLg4Cvf@rXv28T9jQP2OuU0{)KnqnNb}gc{BmW4imE~259%RZ
zm5P{Aw@23tpC=`AXStDb4$2V*%?QnOa*0au?}W+tQf;#49R3}&*)wQQyckBJy)Y+{
zMyS|2g%r~0_H?q`Tzk@3TH67ABWDR$n6_*gr5WBE<;4ZuLT6#0s^KsrnN#3u4d
zB}rYbY?1d!gxE@IhSfSt7svro3wLi1Q9#T=LsCypp|^2EWHVrcp-Nq`GoK=y
zmY1O^WDa>poy=8ZTZmp|p~``t^$>8+eZZp&Kvzr?whCW_8pN?-Y!
zvKl)Lo*NS$0-x6zpW$s$
z7cv>tyCcX@B3JWA(}I|VxS?CWf$orv&BMFnqp-qgKcp=-i`u6>qd{Q`s7yXan#&WU
zYykzRl}pR0wiGI%CBS+xMh8Vz0JALXUG2x~`n
z(>OK#$zsG_>^RZ{DT)q5lhC5*RB(B?po0w}?`o5DY19pT8nQ}mB+Zb&sqN5H=sNTj
zas?bOHNl~`3vqyRa{)@LRg`9mjKmY|$!y{Qz7h2*<>lp4WqF%AAA5%%!L|ZMsg1S<
zpKSxYFLnT_q;>``Yb^W>2T}&6z+UJo>ef{-{q)STdJr`KprZ5g;0yk|V|ox!h{tVh{0k#6MDbz<8qk
zNZKv6l)u8yaj50hP0A`cMLH|BQ@$bxVMdyd^~FY_7eJ#@)QV_BY&*6Gorm;P_XEFj
zP+6x+>TF={HOPB~0-UH*JAfxMQTZZOk^T|41K+(|t%_=}^DxKW0vB&Dd62S7%}2c8
z7mUGLVg0Zhz@zO%Qjtvc7~of%vPzkP9LFr+=xRBiqsE;4l0a
zQ1eAV?G|JNu+zg;6`6@ya0)Mq=U^+)JL-5vFMkskN^=ws(hOQ}Q*1RjHa}rwpnt4X
zcfo8?NMV(+NI%pETGV^2GP+-_1pMkO^);f$s}gTuw&Kvnh^%~;dr20tEVxJ)3q8b6
z(sH>XxLD>wZC)>gxkGFma1Q19jY2t?N}Tdvz`I32(P;wA-*oAz^h261_gAK?Ly=GF
zRArakS~?+A1Q*$2@u6f>?yBzr4ITxDKpW+jYz3^o1ZEwZ5`w8^mC{8$2(GT8=ySk;
z*TB&^3p|)Lu@J=zYqusVo{
zyg|ocSFq`Dh8k=ms-j!5JnR7QdV`6+#8P}J%r+|4wWIW+I1NLhYb_Q9U(3
zsSvrCxPdhXOi)Al0m(aP9HF+JWGUk)d;5pRL5Mt8#vb7Ke42H0*q3o8V3UVT)C8<-7sdJxQ92h6kEtcZ2e^kEDj?1#8fEC;`lbg;-7X(<(aLP{?pxUqpAGs!Mu_#lS)gg1c0oYD=}!klL}@HQJ}z
zwYm|y;@T&gV%i{ZXkn=2M}Xnq0Zz^jn&vuAx4+}{mIo67D+GVqu@P1CcNZ$VBdqxZ?+wG9`wFB;5;aZnUkMno8ky*ZEk_=zFV(Y1AM`1D7(YgyCcl&AA=Yh42mTbf4-CjRr9hpA9mI!V
zTR>qtsi43r_6L8|7N|}^cyb7vQIxjGnKhCRm0
z6Z@f~tJGDh8m0T62Ngb3DOS@5IKdCvVLD#7NPj|qK)+RgM!!oR0T)bJU4ky6%hBiS
zvka-=!tV`QXcf~-(2@kR-}2G2$Q&>oG$!g}b*FU0^aJ!ObbWQzboDh?$R5-~>Jz1h
z8+?w65L>7d+|Hv$9G6gY~N>I`5WUZY*{0QM1M@qDzMyh;2QHXp8YzhL644cy2;
z*n}v{=P;w`EMT{XM@B}%VSmT~+CnsB3hf7#W@}Izyc?Vzp#0YY^}^vu27Q7323+PHJ=i-aqHm>ttm_88?Ip0E
z(ADfT?Kdtl7Bfw-d;(=>kM+29fn}CSHMBFnF^w>9Gsl2)evLWBXf#@l{fw)PX-3TS
z!
zrv|SFO9h_>kU&%aF<*7xO#eWL(BT3D&l&9<9mW4
zhYI6)4ga1k!R}*mt~R@X2{H|V*Y3^X!ZwkVw<~eT2&B5&L|LSSl>6#Ov<)zOt4LYH
z>(Y$-z}3*kya{wG)!5f?ML)&(-Mqv4**3`0$2A#zq=ObO!X`NA{*x*}TY%nH`?B5+Y;(z3;=xYE@h6e?oz}dgU*V2F5pAT-%qLD42
z5UybFu@C4qQ6KmoN=EKR=;&ERD6hRN6HIQ`3
z+R-Yi%#bNrl^26cvI6!2{eZ-&U8OeM>1gLjYDA3AU_P_mc!#)3Xvt!9_2||x8SLjf
zUhrRjrM%|3Rr7p#srf0sCIO?rx9^Y7=N0pt=i3WDcz^gi1%HRa5f7b2ABh@4NqGu>
z(NAnuVW9X@{7>o$J7~?M`chOHDE?u~G0*ArNQuzwpb}~nSqYJw3(8ANP+;nUI^UFi
zD(nI`XB@(zMR5aBmTG4xZfoGY?pon_7c(a2lKZ-A0l3pHI|*mLbT;i%
z&2!B}&1))=>a4l0o2TEPe+pThsq3kYAqC8XokAw5g@NfG11M8ilQsJ%~@mmA%3?
z?kc;U8A11qMk8mUBfyh$fN9GdkLH3-Z4cUfxc6|uS@5i$fE~A&zInc6AM3s0J?d@l
z3-~^PYiXu`kAJ7XzJHE?P4H^yZm>`ASzn!THdg@Th2WdO73f4zTUP9lDdQ%p78~
z*oL4e>Ok%4#&%`4fOpzNUyRO;hCy>(6B!@LjxJ}`vK57&G763s9l8KL2OM9juB1t_
z&UOur5n_7AyayM}4R@xquJf&9y2F@yXfK#kBn{1B=nK%pt3$S9WXx7Pt~8%EudzT12i|Z
zZ*@AuEknF9(Qrt6n7U7%CtFg>s3Fubss(j}h{xBV<&mB0QALtwi5a|+|Cc`j`tVj^
zpQsZD@I13Tx-L90DEa1i`GUg*7Yg=zzxXN#CI!z1I|Z>|tH1|85jgF?;g1bI2<{1v
z2|WvYqDD~s8Z)(74jgKAz`-^Rc(pR%)@jc#_@C1cZr)RRDsxdm3NpZfXm(mo+>U)NIx`
zDG_+U$<%pmUtOG`tNE!V!?MGQ*rr%3S`~}Q($lOn|1vf)l!Tqp28Nl&tH#TwB=Gkg
zGo3JW(}su?VB;Dq)g=sOim~Dm;idRh94FX?rosk(AeYFB;NZFlzHSr!Z}e^F!Y0fvcedk&MWZ$U@*K
zd(z{We71>@sVMkM%@pGj`)tqMxU+@F#0&A}gyDt%i|yxm1>P;m-q)4@e$+j%m7@b!
z<84PBhtvKH{4Eo$dFIB(KH7uSRN@&h4Eq(2xJ;NNSAh9yCEA(j34Y3Zx&!*lM&9sF
z-_l?;R5qj<|JP;omf^5>m1${Vo@VK7HNsEXW$?n@NJ$Mz#^71VS@5Cv6-2MUHzTSNW-(Y`fN5Q~?t+2&3#@o{Cf*p^=-VNSU1rF~!
zZ&5FoUpD_j?n~Hu8V7r4XR{CHRLX0R{~~`!!DMf1-+2F_!2ADI)Nomv;**q~grdJ^
z+Z=N^VQ=E@;%wr!#KdB!<1fb5jlJNh<{s@z1rJ?3xD@Y!hp~~XnzO&7nr*202bh9P
zh9bJ#)G6>+Ekj=@A(7`M@q(ZgUW*y%O6sO=x9PF|dekZuD$-w|23Pkv6w^vf+W2CRgBFkP`B7{uHC9lOqp;KZErE*)0wKigGjp
zI~=C~g)9IEX%#kyxk=|oafsPSHd)AkDa8p&)Gbw2$HA7=Gj)JEMyaYik*){}_*3jA
z`exJ_O%JaMjR@LJlwp&fw8_gF9mxtru=oe<8z)t)QV-j`TOHf+Mg4D>ipU8
zr|e()@7yc}_F3)Ow%p#}Suav>%a;=P9&8&*4+j}k`h|Yi-LmhFO-`T^-6b|8elJoX
zVQb;}aT`4e&KI`XR)f`Sui^UamOPJQJI0=MHL+{0bxm>lewqs8QT!4fNn1tl08e9WZAa=Rk&3-Sx~MIb3>i}z$R}aPW}xty
z3o+whet93B9Qqh2?SBWm6&d;6@(j6yavo$W;9}1GTkEgq@3ub|eh>M5`1d||zw!4e
zcwY~`s`KaXpBb6wa#Hj6dzbm6p-iTTv>n+2`FCT>7{YQnGhQL#H64J_IE8q|8MhMEssXkor0yBv_zPbLX4>UD85
z@*PLXR@#}yBjzIDqdE?qCdravDlpZDec>wh@@_G9V*+2SUGe%QN|$(;C>FgG-#k`!
z!QQd?v;G1V#Y99<>PoAGfB1N=9Q&58P1j{I==srKP{;d5azaCc2mQ}{8)1JcHrO@%
z5B-c!k=G*{;t$O4Yhf1XpbLY8q5{>PTu!{h%YcKR129EXiOpDJbP3W*c`g3oXR_(h
zBH`+R7+>oGcis)y7Oj{4FzW<(!0Tr&{5$B+rC+sw&du2QLrHI$zA_z2PyE*O+oo^V
zzh$N`$xwfH$V$t(nLo)NA1WI;%jvNWhRV)~37(`aW!TamlZKWECuGI0^Ne$aY@Mv*
zZM6MA=S$DLnLSu3CyOmRHypQ{Fw@
zUcp3bcT-J>erP)IpQ@B0?t@98HWvY;AxB4rYlgOmng#>DX5NhaF8PP@XBTwy?+z+A#c6N#3pTU^UJ0UiBhzJ!~FPK0K_BsDCQ13jw~
zTT^%|WvNZE-$Y%_2Pz-5*1yyiGDxn`qQ;w+y3S*J^a*(;V=zSiV|8`6=2sj8?)8;Qmz)eDlsOc#+>UuU2j?*kbBhaM9l(
zv^+YG+b*tAMxZoSjwnLz1qOC0j^XK`B|Ii>Q+;)e0qkDjHoOa7a-CtD#zQc`xu@}O
znNCp~OvjUgO`+cu4lW54_s{fB%Qxgs%Nq12_GjgoE(hp1DDpj&LmoO_T!FVf8b5lp>I{1m@|38k!O5Q1UFMd;8JMcZEV0q0i9G*m=NKk|tii#Ms>O!`28k?=Z(STV+cd<49eA8bLG%
ze(jWE1_wYHF#wozI&9xqnZZ#ulp5ITdz$|tXH91GmowvPdi+=O=Qe4T(&ndC_l0_?NQZD?>y}XZFi&5{NS2kTvGvg-sUsm4^}O}6y-@7>B1ws3N^~kasOqR1ldAuys;T&?xWV((w2`Wg#-sg-V&o-o
zE0rUBNFS-EXp*l=MTIErxNlN%e
z6xS!_z3Z;ye_Mak?f*Iv_iJ}B##yLST$z|2p0@61_7>*u`bJbHdRA#DABRo4=D^B(
z#UA_#*szX`%nzOkTo1~@t--#bePL&0b>vs{5QA{JTrD9Nrr~;WygC`(2-~8WkY|zk+YbAK(z!
z+S|a+VC!+sU^lXhI8j#AtyqZo1X}t|*!(D_8BgxRwOAkIhH?dTtJZR3ITqYnRRM)H
z70dGjnVFFm!Asutd7rW^nO}bEf5m0Y`c9=c_`2is+fO?`1XD|;-g#Fl)$$=e&GWTr
zdZ!=wuki1|S@(UnI3KF$THAYj&bsDWe(JtZ#mVi&OU*&^HRtl!VMQ+`ohY}qaz>4<
zHFGP!E4``M_!zs@U^uF~WLRq{X;h
z5RQjR$SlS_YG2Yrd@Zgm{ui9~2h#qdlNk(pN6K5mNzCZjccgW=EI24|CGZt^H3LG=
z!VeHvq!=93$0CD}({Qc=w|a0$$Pk(zUWhb_+>iW?UW&Ix564``-UAxU3Hbi2(FagQ
z;I4dx^%oDiXhM8v+yf{3-tqCV$C0k#et{w0rLOJv=a$i?5r)-pZWqIx-C5tg_IB0g
ziaUP^fByYF92j1i(VC+tOEjbB0?#I7M{
zvi}H{O7m0evd|K*}P$peryAbsHrHj>tmjL$(FQ+`p)en3nh?
z(hh1BBf?7LOyJCBtC$$JB%5&QQKc%P^Q
zK|<0n66zZ!MV3VS#hh^tnu;ojO^-AQPYS*cbOfC;I>>{6MQ->MQWKdL>y4U=reQLn
z$3#RgM6CwB;z>}BP6CarJ+6w6M0G-2pz1Ooa}V7Z)izd#3=XON)4f^lLT6X|6zhF6
z&D6>8R9B=WsJAPabqUooD%zJ1ET2(vyDGi5L48Z*SXcw5YWf>bI~oT6LwzM!$YUr@
zq78FBb}3eg4&%;~dNSV$>ZN^?7iY}Le_KLr;4c#7Hc49`S;cJu-jm5Zsd%|;eKJ3#
zLvoL#4GBKs9^O#)ZR%!x4N4i!0B0*Z_D_5g=>EpUpMv|oUDOpB6SY8Rs7Y*YtRVJ1
zijR$ouEj
z-Q~_j4zm3UI6xzJumH
z1r#-Thzi`DJ7EO-hqr+ft~pfPzxY@9ZhFsqv%!OS2x@M8(BS8QSM+Z{9e5R39XRc8
z4X3wCsF62!ZE)^)Otk;DVl1VmNruxpj8?3ErD#<5tJ+`Lqq4ZlQ|+kjp(s`F(I`~w
zly*%6bCv^YP{B})g#H{gh2MkTMh3plw{Z#XCGjyWo%2c1D#4Z7Ept`I>C~Nx_r)W)
z5m*B*viA#$BqwCOQ%v=sNr+lq$Xs-2i(MFChP-%4xGH!${K!
z9_S8oVwsW7pcU~1Cx(|s`otEZYA_A3&(Kux%CgbzP-{WU-8vqDoueeSHohK}fN6lE
zgQN2({v=@~aRBKGX)Nh9F@w+_Hyk?{H0oz?YY4}PJBTZYXNU`kEFzs)NIXsGkG}%9
z1S)y~>O5*YdNoFh%>sH!F{V3u66!gel{ZE6BF-=l*a+7`(?Q$%Ba$1D=J
zrN=U=((k8sPf+vZ>{qNA?3KLx5>;|EZIC=yo+v+%YE7(^JP@pBPo&21ZH1{r_rh$%d>
zSR4~qh9ltmfj1x%6#r|n4(umfC;S!MWo$o;990x+80{5FLYjudA$#Z?^w?^{KapXP
zd6654JiG}g7&X42*Xhmlb@c7>y#nXQc3(H&25*1QGuKMTaq9=uO+%7?pLUq$nEJiy
zvGSf`X8rEE>N;zED`hv;Zq-(mRNX-1)IKowb$ESuLSAG*3>WVZ84nfuGNeZg9dCf%
zN1)Q0vKsI|NDripm$#JnPT3&!@Iv6T%;u=LkA(wdHOa$Mlqs*%_R1%xW+t{0@8=5W
zal#_#k8dQDP~Ouwum-U^vQ9G2(N2-O5dNn(V?`@N0#H5d3U7|J#=OL-@ZX?cu0of_
zXGa%<4rxWm5Lgqq6}TB3jC_q9MmNJcvA?j}F>G`MWdLQ!Li7XB&LpCPsB4%jxJ87E
zq-2VT%qMRky(Vf2zX>bA?bC^vOX!L>;=bcpcnAI);US?0uf(mxK878t0tg=;qOT&S
zVBKVfMd8Qc=g3STczpx?U20ULtX#k4K;jZM$3)8P_yFLXMhip&D-w-a3T
zO5{$o73vf22>Bw-%vA6PB#cbTO{$Pxk(LX~IbE4Qm=f+JfmeJ}7D;NCGC%cFYVVXI
zi6qCWK7vuz0VCJ$yI35!oE=hRVRQ
z!IetH4Ftt_G)jzI3%3kS4vh5g_TvKSK@3tHyBkjh-8}`pHJ%3!iwp5Ea5w$}`9~Wq
zilxFB|AQsssf17XlfV@0j4#B`0aw))yb1RiJjVu10d(l_xRtoOxYoEH*ddrH=y9kn
zaa!y^BoICoQiFf&1DtG^!K%ADd_6Q3Iu|=ZMNr@oxqG-aIX^l&I?VR{c9vagyK32I
z((1ct`>LNRvlK{O`??RccWTLX>+0x=!KyS(s`iZbvv#OXsV_5)wUV8A-V*_42our-
z!K;Qmjtq@GK~-bd!>DYfrn4x5aT2|>K4G+Mp`bynz57rjQTX@|AXDuL&jf--exylkKzuuBTTaDkK+xKOnv5O*F0jF{
z{<2YP;&RY*OvQ|ZTir{T$766cxQWnRUrA^O#7rUX14e@RhMox;ut(_TnD)@wT!pSh
zcgJ8cO!U2YDQKFDkQrfO_$+wfUx63?MW}!1b1*;HCD7Bq()Ymoz_Sr*&d;3Xj<5Fr
zZ0XibW|y(O;h}D~c9CYH`ktz_>X~wl^0TtF>W8XUjnOXG-qwb7CC0|)0amM{3#hP<
z2G@rF19si6$gXHQssg_EdF(#I8uDK%gZY^IO;|4OB)y(sktT@u2=hhvM9oB45njxY
zn8j#G8)<$*r-aoB{SvxJM8f}gUg(H0SOKO2NPW4ixy-Xn8*@AJ8>1U^&>PT>QihZJ
zk!}%*q&TTPc?78iaXH~3eip6`_6z9Xa?o+mBckI};E)YK5zy_xYqK0RJ-z_ES1)3-
z;xFUR;zZPS&_TIS6VOTEBmD>S7u^+o5|t0ir*w2SCJCzr#alUesQP20m;sp6XgvB~
zP+|(=x|j`!i|68p;!T0EI4W9;>LqCGL+91?-XIhWMv!GX!Tsp1Z^K(BmDxy2-5>g
znqB8?;&u6(1v`eiA#JbXRsK#O`g$SZ`03aZ?Lksi+Shy9AQV%Mp3lKuH3SmLZ*Cs%HA}!cw*mY>+rVwp2>l*>IT@hM
zI|aR=eAGhl$t*;fP&a_XdkjOvPQrG^e#T^B&Y`!XUqTPw3iM4C>|y~RKwbbI?FMuj
zdIsu3JP&xd{{kP>4*s|!Q7ce(S|d}!w}44I5eOG20=&Qh|3H5Nbb-71W_vq$9=hH;
z?$|C{zM6U%$%bFLXIj0cP%}&YP}NnHpsH3LS8i3lP8H^z%RH1^Y)w21Qg*p!nOh`e>raykHLEljH=t*AKVyx?#i`umU2F`_ptPg
zRrI5@KU560A!P-*DVa^~P3lgZPq=`0fX_mLm0~6W{caCx5xA*xK}XpLU5;vrx&b^$
z2Qcuqf(I}mP6W-M3>4%&K;O6toOdfwrKn2q%nSpU*UI>}cnk3M?}t0^G$1Cc;H+ze
zTig6-Byuy-6K+SJfI|2Z836s)hp;;lg2Mwp{NwyI|0W;Kch@_`Tjhzt8Gn-Vg#EKM
zYR)nZH~i6E(H_#AP#aZ!;M`QK{Hr*x*rwQ{_^W8Ie5t&uO3*CSF3=6uA2u8?IV=^n
zAC8ai4F7nbJWfUQQ4wkl`UWPBZH@nr&={02qoE!?f=vQi%XUFu(Oa=ZQYB82SS7=x
zT!%2@y7vkP|MrMYt8ck9n!)sWPIkCGjA*lBo#tY(=K;l$IM}Y3;6YyrW
z$XcW+l7X36%@tc`Z@I*>i5<^sb8shtc4}95X-0@sIcQxk_M+YB+*!P*|7%WB~
zdVkt}>QdmA{-lg0D~X>8I{b6s(upxFbU)MsV8PmA9l$p$L?Q7{pz%H&PXaaWsn}aM
zIUR)a)5h4f*dOq3c8Jf6j{$u;DXs+7;lr3ewkMv6S_?kawrD>353ps&g1>QN+!EUk
zn#4b_f=&jdX%c8T7XX2x5Lp7g@IImEpxERGzXrwyIDyyxCjLRb4ql1pj*I2&V_#uC
zXBL~j8BXhu==N)mXe!h_)G^f&Rfeig8B_L9ZC5o_chrp5Zq!`?KJq?eBXd{lKhQ<|
z&wbxFE|?v@f|N&>K!s>Gb~4U`e@xVn{*VVyFVMHJf}C-@q5R!~Hlh(?vUr9_C#n(4
zCHuvV#gE17C2GkB$s$Rygf6)z-Xn?woAC~R9{)2hjfde4h37qR>Wna=G)@BB$Fwj~
z8B6HfX!EI0DB~&L$X7@Uh_eZu@EV|wUPZH@4l4o1?niKFz5r79F;JZ+f=Xf{=)cQi
z17e3^X7-Gqh4U8%MSu|xgWLKWtgzdlw)7po)(`a-RR|97f5CN40T0JIR2u4GygRTY
zM!^cY1c8j$
z4quO8;;&FnbW5xTH;B-SxQj%h>givY4EAfz4&G5dS8z$NQ`k~8LDW`67g0ouMMQCm
zm<=;@l=zisgK&`G5?{`5!o(jR<=XMm*r4E(rccw=8hFGs-v3a;z_dD;g759nufNUSCn
z2a>{6SV!-|IcQQmC0+%LLoqlu?gN>5!v8yTSEHIhMRXIK5yIdrZWr@JPlF>K5BD-P
zG6@L)WveHskqE)dpy84O{p7fx?_U7C^yzTAzw0c3J^q}gyR*)5{^8B$e-Jbkbrnw$Cy3{YCrO@4nn|?cP7=3dhIEE>sC2$G
zUphmwL_AT{PB>HG=g)!pITh3)gLsdC?i1nM00Q4v)^jF`*$ecEKWGHn0%{s{9i0H;D7P|#8Eh7SM@+fASoG=)F&fFgQ(yd?e`DwgwODY0+RHPc4=1Dib`
z835c3Eog%cfnM;Ycl3YtCHk_xg`Rvj-gVu9=7ZXy8mRIq
z4=T4PzbaKqgL0rMM_ml7CtGLG)#F$}n(%>b~f^~?uL+4_#_z8rb
z#C@dspjPinpUsfNh+gGw;I|O;5OPEbqI<$CLW|HPj0^t}oe-so`--ii)1sH6XQDo$
zr^4-mU%Y3qc8=x@XFp**XI*FAWgca;XIOyoGnb~L(x?o|7_tS_o|(WPnNAP@bGt2W
zE%p_7hW4N?#=8M!Gcz_8dKWqHX0n0wLXJoxQG^Z-oV^hNI7h(018n*n@Vrj}?f4p4
z5qg5tr#twyr-S-^BzU-A!6CG8qrXMGuDElf2}v>vk~&h4IQ{(hlfh&GxJUaJMT3jAVXbMj|O
z1KL`;i}8&0j5_bj${@7a{Zj46M2V
zIMF{t8DZ@l0Vl00;QzNr)dEu>4V-lU0g?Y>bW^k;)a-UddPK0`MdKhl!ZJ|i#euxt
z9QGq-po4!qs8otQVb@Nl*FMO07RGa&F|6OCYpX5L%u{PsJz$Stshp#n0M7+w87Ok{
zHAL+#?H1iLeU%}@#IuaCJ#koFx!%wIQK6SeLaZ}t2xbDVIpGhnJ^3P~9j!T|6Uzho
zY&~xZoIdqJr?8iZBwiw(1uNzd@mBG1c%F*u#c#x;#ScZ_g*5_Ob@`pk6nG+T|RmeM|--
zZyO+WT>%$Bdq_@5g9@K8avpLfRz!)wG#C;ug*CDRkR}+I$(ZNhVrvbRy?QJKM**HT
z4(G*susg6~>?gR(+JMZFi@pcGx8>l)GsaHBT3QYihBx5wRU-X??Y}OZ3Uv1*;K-+k`0pUk_!@|q!{{|gM~8$yZJwWk@gEXT_qej
z+r@kUE9OL636(?n4o(3f!Hc7T-)au#5xNll82+cD!7G&zy9~4&YvdGA6Nf}vMsndD
zr$WL;(@1X^)7(fKP<_@!O2F$iF2;nDx-LEuZKPl22Sr~p0@&^aN+#D2e0BQEF5A7p*
zrJm02RA40kYbV%dTHc$Q8!Pl@bo;ayG$wT;^-R@U*xkiSw*n8J^~z(a73x8n?%GMZ
zW%|>GR;K=z1Gb3cjC+}HaL^boi_%fGm>GCGu_Yyu_L05|X!%Oc8s2O{578L$TQOBK
zP_j_cThdgrRZ=6_Eul&di<^ikVvp#9Xg{o-7lj7|>-YzGAGnRVw?X7r&pO6z%uvB;
zvlzNZ0b&hdFTNWt3!8}1fWz=ktTw8MTts$~Bs$#ClL27b7^m~Oz6@na|8K0`W4U#R1?#}CK%
zf-!x7n~6i?7GfiEdz`p)g(M(ZMzp;Kv{i^!1`iu2MpZi5SBtLzkeYod=qq5kM=>_m}!t0ZmLEI233bq=Po4
zLl_O2GrmYHDvlSRGSPnYYs^7xbKHI0P&|!rnJ}IxC7mKAlK&;2A>+Ysw4Jh#vYWDp
z(uVSoJcdkxT$~0_)BjFD@Lt?c@LD3^wN6Ck#wFk?t3@t{mxpEr4+LubaUa#!0D9w(
zpsGB@+22uW|7&e-S!?L|)N%BN*!ZYb9$r_d4nTMvJ=ET0i+x9-xGx*Hk(f^h|U6T^?6w_e0QIxAPqIWO_qh
zl0O4zw&z1+WGuK6pTrnYKPtqo1qawZ!Up0LQUcjU_EC;fiL}|Yi=Y>|0z|(e8V}ft
z1ll=jkkS}fmr>FH7|lbFRj>#*1?tQBpd(R0#r}KbJ=B
z4Gycl&UV6@Yk6R5ZQQ5-q0QD@Qgu+06q)rq>ddu+YIAD4)vm17)aKPKscTcePEoFG
zpuVG-q05Kc=1+66&Fe_;tn=f;8>7EbZY-DBm2#M_VSVNf6Z{d8rSSw=QcB9J)M4_P
z^phEPGqz_82K)cO^fvOmG+FAeQrPhe$1_je7AM;%Ar
zO#F-=jRRi|dJ(ELej{o`62e1+pTPk!!m9yREYB%+6xb)(TH8+Bg!WVR496x%x|8W@
z=f3M{&YSE}yQ-Z8
zXH&;~`w`n!>l4cov(ebca7S0KA*m-R$qId4thPz*p_;BWg*A<8*40F6rq$x>==I|j
z_m!z?m*%!E&v4K9)ZE-gbGGul_0I^CV=K@aTxZfPs+%F^4CPM|Wl5jP{w6I=nU*#-
zJ&-Xrt55dSY-aYdtY?|73|7W3`Np(ysZWz1Ce|hVlH3&O1Y&+EcQ|_jb2D8)yF%fT
z7ZJbWE3q*cMPFy?v{z>GbWWWVhHF}y7St6%~n;VBB6d>Ewe^ZWv-M}9K5;~IWth{#c4}Z3X={c2qmY4OZh9fDQpXMM?3JbD~B7kBBabRTk90+?#$}V{QHjhCoN)q)1e5qW#Um)Gj;NJsqK=-$&8XYqK|wLw~Y0P{+_BK
zV@M?gG43z=Pn;ZU5m^+j3ib=Ad=tHD$Y*%r5ZS+46_%*E+H}$Q)=+A2=}CqWhOpt2
zQE4i({DGQ`#X)f&^DYi-3)e;^=tNuwu@xnc{*l?4qvK5y9ux1EHkK_-oRXBAtWBPg
zvN?s8GA;RA(%Zxivinj*%n->0*}MpQE%O_VOIbsd;GbdUqMF9c$kR|w!0P+$`R1y2
zQ0!x@M)OhANVt=C(XZE4Yq?s#+5^=hxzbbrs%~a&ZFQR}L1n{=qvf5-`;}iRr&p}4
zkX6=I?ytI9y{7hOeY$F-X1DIL;hgEYWwD*$&h~!{SH(Z#tfW-XxwPgy<1ZEOm#s+o
zDF2i>J!fRzulz`XxM)T3k&>q+f#T;yC521#U*vwzzL=SxJ~VY>k|iNYViNr1lG#fc
zxwLa+9#M+#1%AJi@x4(IawfRd&-Y=xRqj)+xz2Wu7k+p||3G6Lpk_^h2yQ+y;XCB9gSXY)w*P
z3M%zsYED|qwBj^lYQxmL6hqRq#G46UCHF|2ay)ZL^3_+(78`0L2l@L-6+
zFnGMKddD-{J4>ypu`#H(>TtR~+TWVV8jHGvdW-6flC7MrI97kL?rQD+ny%H&tL9d!
zD|T00t?*TJs617fUxlv5)fCj$)%g`I)mydi^uJ(E5^dWZ=iCeZ93%(Dz;`8^Xc!iX
z+e7eCd{}lRWk~w`td!i+{O<*!!kVJH#XC#Blyoj>SvRfrIRyc8Gq7;r(cqqOKw}!P79E
z;zuGa!s1|!@0@3qYo24NZLVb+B)#x^(ZHbPDuAaUZp$DwN_md!$;#%bDa*Bgq
z4yTHTf}`SD2}t6Ilv!y(IWyxz#N?e}tX)vEz4}{Kiz;j7{YqWsx~iONceTAHscwCJmXe_Eq{)gZBt=Xzw>Ql(#tjz?li?W#d(8&JS%cl6G4?YnEL&{7967FLo*%w%K}uwEJP-Sx
z(1mh={*1MdD-qa5xCCC(`xKJ=OL|^rVb-3k5n18Pp_zj+nx$8#2~$5LrOGBtHVVh_
znCuPoI&xb=1?Io_^2oT*5`VyR(lyi3#HKUrjoF4xxsF
z{sB&pA#S6udw4>8F3v_W&?d81at{c2k~gxIDJ#+kW>3t!UeKYae{uVgjM81DlS}cX
zIVBg1o)%onTbfgsSuKB^GA{A4q`mMRHn%ggPmFC0i}Vxpd-Xv*$8bR3SU+B0r(bDk2Aum7=62R}JH;t<
zpYYxYG(l=(!!V!mDdf?#qs-f!bNm+~yVRBVJf*i>n$a;6mDK{CaAwEM-Wf^hm(#wa
z982QLawHMKX>LCjiGGsYoDjkI?lD|qBlXwig1sk~YSsm%qDBp=@Tn0uFuZ{8}0+DAJxwCx3d0nbjUofikZqW4xMLu}nX;4Vn)!tF
zlKq*J<+<*=6BI^P#y?^hgh}L|v=mks?r_0G@qmP~q~@vTgBX#=>?@ZS%pKKeQHJ*2YQISdLV!}MiBu0##z|R+*m2OU&nMTMQoRgAI
zEG#SPU2H1uSF*aKU&*WDFGY%@6
z6@7)A2%Q1{$xv@+_he_j(
zqNf;~hNh-V(C3?Pf8xyaeDPJnDyfYx#+nEzlrHo!tX16Gf;Zxu35$|kDIEFr^eq{x
z3}43a3`xfIbXK}0ZCL8$WKm)xsYiH@*OvW`zK!xPu|2Lex=pMl(l5Bz7jlnw673YL
z(A>pn(ZAExY5Qss%~?%P4PNs~JzI^a4yc+zZ+E<}tin|7syqmK
zPC~V+`c2KZ+ECpP(00YtW-X#0W~#T8!E18FTOB+R-Hd5OoK8hC8*!HM4Wiu%hmtSI
z7iXDr=M`{@vWqz-ze`${PAF|(daPtaF|{a=KQQ-n*4Ok!soxX3OO3)sydms~jCRyg
z(j>eJa|$&$#*IkA<3N*2_vLv`yVe1-ztZ7z%yi7OkF+(jQY~IntufK)F))mcj5y;&
zgGY}t%!0&0r;%@73F~(U#}OCV`^oPLb&p<0$-&YEv)a&qf7!L*HENiq*H1qX`PxCnpBNmeM!AUJw)9~ov1#fI-(q=@Yb!bom6wE
znpd4teZIPB4YAf)`=#ze{X0cxl|<7>cfb%gl~^a)&Cd58N&t&|j(@`~As?V;vS;%S
z3q6v}iM>*bGVIx?{OrO5MTd(Ul+xQoVQ&`d8(8ksdPwfB&f$=IaN9oCzRK3Y+St;`Txx;>xM4Y@
z;%Z!n>$KV*S_^da7eK%CoAIN0vQ6mx?LO?=A4Elt#yevJgiRC{V=LRpD-^Yt
z5)=0%Yg0eV6EnOSQ#1EwuFM>tiOytae3r*kPbPb0cF7816J7>8lhKOWhD5-hK@R}#
zBqdnpd*RM>+HGM=ig}q)Ubw2ban%J?dzD>zOW91hM{yIhntf{<
zz=?8ARiIK+SyJ_^YGrkP&DWX>wSVeT6mONk)pFe)7|Sl!+4fV;!JejpF!CL>1n(!~
z7>C(WUR%*lX=ai#bx#H+=U3jl0z#3osIvG_No~p1l13%+;(LXz{1v&|vp&OJF)8_8
zLM!oees@kOvnQ>LJfENk#=*?^yl5HHJ9I5z@L4^@?$=J6+EVRrIGdl+ml#`{C%{^XIJ{Y>=IwM<+OrgLl2fV`z31=btYWUB&4m41
zhdCQx6B!&@0sHrSS6jzS+a(JDcD^Uz4EE@T>z--_+Rd7vdW5=4wN#a;x~lB0{G>3}
zf3MqL+p8w0y0q%Q%0-pyVLW$L$*P}JKdQ0T=GDJe{8Ba6uG9Z8$}RJ4en*Xap#MU6
ze!ML%LK;L5u(Eg~g=ZuU6XmJF^j+CE^Hc=`il!Hbi;GIu!+mjm$+O~xMY{@0@~&j(
zWiFEsNYTqSOA3T2UK(4*sHbpAckm;z8E8*zIh=%d1S9^2zM-B=zz~Tz+>XPJ9`=n^
zrscb7xv>D|L=?`EAK;N{TWZ$AtWc@zL8BYc%{1sukF0|nI@ezBz5oOH1Y}1Fp&Pk^
z_JVbo*GaTrx;T*ue&eL{4H>gCqnV?#c4l45nx3^avp8eCJTvtK+`CdFdBP0dTy~7n
zl2$}U6OLkf#fL=>hv<8m%unn
zu1rvh6rbwm*ZxyeR{f}IKvibduqstmtLhuoTWh}5wypOn-l<+{U-;sXyx#X(mvH*(ls!oo7z~~+4?(9xVn3ngEvVP
zjiUPE6vT(10`J0oB%n)L$;iohsXx*v>FA7LMkKQm>Q&jNbv(+Kn$5%pMg
zg=)R3k!pu>E!2cg)Q+rKQ~k1vUUj%~Lgj|aib`%(m#V8(ORGQCtgh><2rKRCJl%4`
zb<-KkL0inx%Cj;sEJ8-h@Lx!aXk=DCx0B$cxU+0O@}&GxQ05G&YVJ9(#-&41Ery
z`(JyHdUm__fhwxfPPGXvO-oG6kCj2_o!nPvXz%NK8Yrfq
zxsNT&S>kEv?-<$;Ph#*m-m$O
z(k7?OP1+`_krqiBi~Rf(Tq>K&$fhnNox|5)AgL8-A?aaWkmYxJ?5Q%Xj>J_$T?#wcoRUkN7?A_txJ@e{%j#
zDK}O;u2R&<>(?mns4r@t==pHxy=kB7e&X+gRK_v5mBfjZX7v6n46l)Be!{5aOKBZ5
z)@H5D`39p}P_Vjidr|Y^=EWn6`~@5G`{ikK{${n#*p%i?9-p{Z`b9Jtsv1JJi206Q
zP34f22{(XL%0r1`-6EI5!$Qr17yT!^rS3hB`L=JCH1h;wmHwM9U)NQuQukI}QOvII
zQg5!`tWYTiEBB~IX&UIL2BB$!MPP61I^;bV7z3TC{it3zG%1ZrU_52zawqd=3tNb*
zB~ubQB<3g2O9`jwQ&y+=lRGD`Oj?rIUp78ru5^M#B+eJwd4D*ytnZ9xw9Awoq&+}h
z_=TwiH{p)R$?)^wd;b^jb@xu^3(#b515(8%*yX?I{?-0c<5k9bU2UhDPgScc2UK|e
z9{=v|K9ay?%!+Wtjd*D=`|^J9Tb~Y*EEN8a}8HaldUxl#FH7cAzR`>
zOdS7;w3hmsagOs^@B#drTfi+aDP5PbIjd35qg-@;mx6wUOAB#@!wVMY&&ZpVo0`)m
zD>p+bZ=L!H{91!#lcg`k38ImLX}l5+j`f}XjG96@K~fQj_+j9QT#qV@^+h%XoBH>7
zn?bGSl9TCZVEfl{&7?IH=u>rk-3;9^-F4j$okiDN&oC@DCYj$@cG>ni-n!x*p1)gA
zAHEcwjG|%3;wKV?5%v+Tf&8u_;T?fcFc{JV
zS84jXd2Q4l=nM*$2L1k7YF2?St6{|kAP00Se^TDO;$_8}%EeXdt5r3#>k<{8lw;JN
zVJ^h^k%(ngiNlkbsUOpF(~B}b
zWyCWq8Ba6jfE$LA-ay_C?!=np`Xoc5T1JD}F;U8p+!M7DUf{Rn**M48>sVu%JjP2}
zGipb&m{@~*jse|3tamhsbPN{DDs>?>?nIL+c9gB^_yjhrNDxLj}Mk>*88@{
z_HT|1*FAR!V19fKbPnwcOCpD&&tgyGAAn1lgT0Ky5u$|4#5B@&QWn`pK0`^QE~dt)
z%V=k57ieE-zi3}+F(A_=(uH(Bok{;nFx69zyjn;}aEz}oPjX^DQ8MIu9%8tsNN~Lmw
zs)f3zW}dd2E~NXeuQrsLJW!9D2N7ex(vdKUx7}cMzlbr6c!4%
z2rlz)^DNv#ZZA$JHjQpQz6Gi0yNcnaKpC-y5x!qx(7b3Qg3e6T#AtQMnx
zf`VxoXc`GId9-b$Cz2HY8+-uCeSRMnXe?QtR_<=Dbf>|7&=#`vHYb@H8dD9$`jxtX
z_Kx~dYYa-iusTgX1h7FxK-TAJOckH-^<4fNRVzgOfW$(M=(~v
z5tQ@G|3CG45?{fa%**G+Al+>eH^e#4>CAZm307hjfmuse(UNG{)Ch1sh7r>UGCTu^
zU=-*(sJGxxeFfPVRS|MzK0-q(!neV7IXKJ?YeFdCQ0YV8pub)ao(!YA0KUE`{50$c
z+rqCQ`_2NMmZOks{xtkH%m5E27U-^hAV20>1P9uYn^6&@eyonYk0nF0#Nznu_#W`T
zM&o}$g);?yCM_Y=`zGW)Pm4_j{lda1Cwco&U%
zK$54!cGTL_(%oz^Jv8++*^GybosDuM*QhmIgXf9ihhZ4hb}&FV8Dkk@ooU--f8!`~
zkv!YI$^KyAa%cduDq0bDqHC}h@Uw_?@@$HmI)uKHVP&>u4}x^(y}T4qa4Pv@LDSt;
zxLHUL4Hoqhbr9u<2qKJ#CX$E>M7i+03gL6%Hkiwmf_;J_e-FVjX>7W&)B)x
zQpi1N04Z$WqK83aGXm(P(?K65j|L;)c>-`2uSS-L^7icP$CZjb)x}j
zK{>JPSW{5r^^W0V=BOW3&7o)jQiRP>6R@3sfQ~6Q>We%E6^a>Ig;0^@kZ|=NG%yqj
zo(ZM}H$oEVHh;A*A9!l(JPX{TT^UZFJzy)gEww&{yvRf5>E#k%q%gp%rtYE
zd58I`Ip1Qm%(DW9#$N15bEdd{v82ta4Td82czfX1JMr5LvW=jK+ANR5~8lA
z-)3xJUS-{559P?Xf4N1xWxS`*XWRrcTL2lU-37}9zhHm)Ab2Ej3W&mIf{B6-0z1D4
z{}fLQnvwI|9b7C|!B#?s^L6H5#$Eau+BNEB$~p2<(n#V8$j=^#tHmCIoUi$q*T5{k
zhl)TRNiOhr(da@H+^xaoL5P2fZ3B|_j%aZ-4H8p-M(QKiVO(cL4ny)?9PuF)NG*Z~
zy;Cg`fc0bnjN4zxWljJU>0al!Sdq`j9AqnU4^+%Y!n+~Cc4*iEBx6_bWN7GqK|kI{3{bjZmbfa--B2VBKj@a7BSYhpG?)M*ZS
zyi<^{l?LvZwMYu02>%NI2|obk*YPkUll;$Diul55NPEzOEdwP_JJ3O{1s~!G@O6XM
z0d!LPz`xKJv{RHw2vI}w>@(yoau>Ob97o0?rAQLu4;#WkSe1@J4sAx53+u?@P-93c
z6o!5T8$oK`+rabyE6^79l9+eAx3j0zo$1m!&e>1cK3OT&{+2K1J?7Qs+`-k_LJB!;A5+L_;{;>J%VX&H1Gnz6s&@Y1rry+F~Wi@#-=>qWuWP{}ZMPGy~
zgFNO$@ZfAhH$(45S>g*IF==}&0(nHmQ5z%`9fUWY39b)p#_Al5R3R{Azam+X
zTdfJ-LdqlM2nN{{DT=;9)+6I#-X23F@Kbt&IFUw?bl4ZNVcq!~NsF?gS0FLT3K``C
zATg;9y!Hn`u{#4?5Loz`bwwVAdxiz!+R*FJ$Iz}&N$5{-d@vXo9k}8*`NnzcK^b$$
zUEpr$qB`0*+CXOKZqP^Hwaozb=rl{zqBPGo5ljtD>rK_BS?2bZ|B+n>+g~{LIwt}_
z^RRcNpB$uw3d3t7%i>?r{lPuZmWL+(ix($_I2FiThk>>Nk}-oRVRFA#JVZV)XJ
zb0wQ3`O>S>m(st|jD%(hCg}pHOEOOK4%BE2kw6&dXYsf5^x$~<%^t&A3rVx@=rd>+
zsXS^SWf(a|+CjPv3@IU@3w{CaDrk34qNkun0uO&1BvfD`osjq8DUdI*G87Gd4r+j4
z`~z}giy*^v2KW;5!t)>}b{^6)@+U%ujHpY|V6-=6QaK>2Nds3B4aZ9MWuOqDAOkpr$rJ65U|fF-OE6M4LiBOb|x>Da`Ye
zA%5sda6<5JAS*E4zr(l5+s$KzdMV9u!Pd>1U>O3v?$^e4P%HHtjzjH!n!dCCjeaQf
zlqwCW#`#8-vD|dga=}J(taAF?E?d2+oHdI>-19uhgBCI8;DQJ2+V<1z&
zKF6W+mhkrrdI^(6d&R|4n-rbUGT~RkV%ZJZ2H9fSNLe@8AX%DhR6<87LGnn{LHK|_
zhBp&jdwbXh)(X};=559ddMRxJ^#)}vxh1I=@fN`Z9`DuI=Wy=oiyjR+vAfX*5g{@m
zd=1=QpMs$vD)bO?tQo;_U=iShV}e(MKZBgm9LPQ{f@JUuVLoyR=@)SToAz$BNlX_z
z9dCttfog?5iRM6lKr!qFMYvzM`9K}cAe0iCgBH6Ee+u6R@4;Qi&BqCGr?92iN03)r
z4nCn1kY#Zy9*2|5OV}AG5gHN+>4UC7YM{S=v+s{r?w#V<>E7y^?riPA*>_vDW{GLB
zp^bj5?g3O&T4^!bYRz-aY9R9EYC3ApfQnM7?W|+y3k@fYTg?ZppX>{upFBKp6db#Y
zBj01!(M;Sqd?^t{zD&ua9iumA-eLV@^Wd(vM4%Ub6IF`0NqfmUC$3FAm#9gMB?b~7
zC7NY;*~x?z30mm^iAmG|ST#d=k2o&Yccy?j8@xM<=yO2j)REGYJdV_vm`@?C4n!y^_$Nk84+a+~ha+kZmx+CsH&p?mL)7N|1ONCs~4L+>D
zF{lKe__45d765^DAMj%~KqklEXe>sD6fgyNyXIkg;aot>rx88@FMmFyOk#=m2@45P
z{6F~rfF^ejy8_!CtHyMKb*evF2l*^B<7Xl7ju%ahAmRI=Mc_dj;_u{>c@v;Nztr`~
zneOc5Xm8KAeYbRhYG$fot&XFuQwLRTReO~w$_hnS@JscnpIU#tzMZ0lvbn0WdZwnc
zc8>0rq0Icv`qYu`+2GF!(U9WkL7?oYu$u_GNJW%RK(QXgq_7Fx(cqJ6BkCl%3<}Ge
ziJ0V;DMwNlr&Xqf)83@5N^74sAvGsOmMl-2muQq7OL!qE72grs`D=KMxX0PISsPeR
zrj1b!#H=D}7R5{2L`)#;$Dy&S(L&ViSZ;I`k`o>rniuR9Xz%~WH{471^1Z`=tfBYL
z^p*S0!&3vF)BRQcX#rngbC3YZ(){pCV0SNzJcwe!Rka+o6MX@*6gw8)zk38B@gi{y
zWDr~IQ8c{`9K#&pU;)er0;Vh;W{S>t&K0L;Z9zhDi
zkAjJTPmqfF!t>GH*u5O;tXmw<>|bnKtuo62Q^YVw@6oo`Y*ZnN>iX{WtonC#6YJ!4
zziX>&Wpy3u$n|p-8J|D<
zW)Jpo?litqa9p%fs*}A=q$h7p-74>u{wn=d#=n_|GPb6-O`jrnr~XKJlKd$Nm9$Ux
zS^84aK=NJmKp^3F=l#dMz_GAdaI64cksM5EMpO_Aq0ER9_1sf5??(Xhf
zbIrB8yW6$9ySsHQ1Qk({?vClx-~In~u7O!#xt8~wdEY1Q=Xd++7Hfv8>M1tMV(Aa0
zf_NcZ=T@>unWtc93~>cG+c{jgQ)}B#+t%CiZGG(X>~-skLCv1J8^fxfgqU!u6|&`q}fgO!8V6guuirm+EQ%0
zZ8dDIZKVC2y%ETE|6|YJb!~SCGTY$Q@sf)XdLrjuE(H-T;um=y-XSYw;c|zZRUCrv
z>^FEoELZ2Lv(yH}kf%YZSfok-QKg^aICvC1&VYXKH`yvJM3v<)ei_#vJ|Qbzt#O^_
z+RxjrTh~;zsdS>7DwMP@9#yzA-&{lFsYPPh}Sm1TlVGl1$uMi8T*9PT16
z74{(ZOJpB0i`~7z_waUj+1uE9SXEWKE6DOKrM@N6Mf!rqd1rEhvM*(J%!oo?{YGkJ
z>Zd&k1cgh`o8r1bbIFhoXdIZ3+UqcWmhZb+GMWAY$i8Rm?{k-
zd(b=OE+waq({9&gd2BG2nz#G(@H-T+C&(*gLTFwX8{WOzv})nie^sAYy)QJ`8DaNA
zTZZO^{0zPxWDRT)xFX;=RN2dY?^xPdTs~ubqP#DfJD9wTg69Q~3>^jM0Uy-@MSILq
zYN&lgDWW7%B*ZXbI6s%0$8Ke8h(SMspEA_-)5$qnJ3<^y9333H9U4@RoX!Psvrcx;
zVj@A9ir`bg^Xn_jMAke74GEG?rZX~sAPG6H=C!-?cVCKOrQ;uKm%iKZv1B!Z~
zcQ~o)lzpgcA~O<|jGIz2$
z*`_B(wPC7km>MRHMa-92-yD{^Vsmm^gb7UIpVmos0xPN#RGUqcwW5jDur0sAF
z{!Ff*SgI4;erM4Qu$~FjMX*3_ko~~5s!n{C&cO3^ghWXjM29dzxW#`3TYfaVlR4*p
zjM|0Tanv@;TBB-0#fq|*B_74|K>?VNXV3YSt;AHuYeWr+87nfR^b_d|)9<9mWW354
znb{|+Z+2u(KTNS+FSHfMloeJ?vFhwM9A563Yyf{(xFMY*XVC-YdzD^@&CX)drK9Pr
zSBOs|%TV8UeuD$L1x^cm8aO@ZQ_%0AH$iiPyo2%s^YNAXfu{nd1%wB@^FQoA+27#5
z-tP>mMw}(VQtb1{`xmmjTINH>xrXDO8!)Nip*yQNpnk8)Ru(I6$~VXo=pZ@>bpA@r
zq3jkv3-9>vsEX}nt*F-65c{rlFLnRp?(6RE9`C;HuE(5V0#RG|j9MIxpNrsna(lSv
zoSPfUzvNBA0wDzcS8Jq};0d_Fv}lj4Vieg0-dD5msuPGHXW`_OtEH3kN
z=7Y?)nZGkVvu0!&vQx6>=S1Wl$(vL_7o`@rF8fqbTy?>A(2?b`Gksrca;rjm6{R?D4!3{nF6U2C6-7_$2
zwBJ3=JskVVS$zMIy9RTFd563D3G2(P;r#eL{BNFxlUiqCj<6CHyz9aiVUW;T@E7i5
z+N&Ag8Z&eX{xi3M>%@ud3Am}Se?BDzZu~b?}o}*vNhbu*upLTDQp`@3G9eImiG=>zVAG6F-r5k&Hx;k^TlpNkfLF2?bTjykDlwrMM2wKq#m?dt%%cCoG+cEa+;u(%
zm7IrMIBJ{QFctTdn~gg-3N?y!xWW(P26L0RD_jX`7mYyqTF*^ym6het=`Pwtnzpn}?kUPa4`PnA9|
zUsgFAdcl3pW$ugYIlhf3N;xEQOH|r^sP1Y8>i+WhgsIo%Uc7e$%U0iXzo-6F0^SEC
z1oRH{4+;x<68Hq4PX(n1T|lNhIEV>+9e6UZao~H@C;km+8}QrzD%N?vUmxGAmZ_E>
zKG(gQdIg$am?r%HRM!;GMILGT=K3bOEI4O>cl!w~UQ3V7kte
z?Z^&-??eFmhM9-GrZ&h@f;$s?%o@-F#^BBmgKO3s*KyYvml3DwS?6e{!nw?mkKE=u
zm?3?Uxm~Kvhm!St*~HQ|CA#8Mg$)bHe6PGdxhJ4ATbrGoH5OXE&zbKszh^o!qqELt
zbIs-+bzxmsGW8ixrY*A(
zE_M&WU9E-5VUDT6wqqk$HJi))V2VJ3-pNd2+As}JEweC1?#u3ZSn0lQ6Uc!3TuWWu
zQFlmlj(3(j*1}I`mHoBt3-a}|l{=t=Yh5NQeE}^+x56F;`}4o%IdaEB)3++e8)x42
z>}}bnvIBGO=d{Yr%6*t;#}4tV=umNfNm5yHg;-T!+vS+=TF(Ta1`#bfq|Ibg`n9Z<
zB2u*+_i?aC5V}s!&06mS?kUo$mX}cYt5EpT?g6W8j&8DC%g-{YUsO@bB)Q
z2lj7!^!Nr~oh^R3zCV3Wf-f`+{gGO5o9OH{%KX^mV~Q}=Lp3wdbEQYTp3*ng?bTW}
zDvhM_Qw;%YK(A;m?nIekc3`i%}c)1DWn0
zb|TxIm6#Mp!PdgQUB(#kIR;$Gx}exSLN@Y(dBQAYT4C>wcfZ8E_7rzZw;C=VTI>lQ
z;61q4H5?w8`EW6acHVN-fnUWidpfEgMb-({4^=@>!49im7;+$DIouQwcm9s17Wlrzh=sY%WU;dAsKM7*5vG7}8Chl=X)rn_5#H{k=7wQO512yXLQ6&%8Gwt~h1+Z6UFCc1y9PBi7M^
z`?jawOuuPZ;b~ag2fp*rk1Dn-w#)*5$mw(4XEL}r`@L0I<%i~uSm#EjZN@}HTf>?NykuTQ~>
zH^&{VWzyVkWZfw^DfhbjxxK)2p5Q73RjeH-1U($t_PzF+_NBIuR?=D@bp0(Ao64tS
zmSA8>C+I3y6|OEw&0n0aK|eJt&+z|`VR^p!W%-8+1{C%#8e6=-B)*g@>s;}(-h_md=(smSBt3X9ir+=6Vy}gS~c`
zUz%!}ej3LZFB)jWNzWCY?L1F-Bs2#LE}KXPv_8
z*_+HhW*ycw33u{o_cQR0SGfJ)E>5`%UHKqBJ6-EtL9SBgX=iKr6wZND%SKe4_S+iS
zSk&D|Sq;|jRVS*ZR>`VPRYq4XsCZf~l|_}cFO7n}rCL!3C}^+dN9I4tYnPXko19yW
zXAUN8cjg7;Kg*w55Laj_ibj=we(8gbpOrz@hgOKaUM}Z{zd1>
zmMKD1&(veJ#kv(9F4Wp|COrsPAG|Eyb-cHDfAh}q*824JIpgE-u~-O8fMuLznx&B?
z(Pxm4%lnb{0`Fk&c(0vale}7c+0Dnzz068;ylEsH)^-?`#<7M>&$*s5oP?kCP4qV1
zZJk9oP`g8ure3a=sT-@-DU0CQxd$Do;b0K#q|3l12%?^o&B>SOM|MNaO@x1193m<2o;~M3Y>V4aLr8g9CUPrxp
zpdYf`JOm1YO{R{fBI5>QE#m{jY(q7}SQd*^I4@PRm#30;>KrTaGKFVHI#c!a>Dq
z*$n0w_WE9o=)UJ(>mKE9;Qo%jQ>AmdQ|U}|yl`A{Y;!brT($SN``Y)~@~kzn=QOMw
zUU9TMzsy`VwDe<1(~?WYLyDUf>x-WkEiLL%)Vye9(Y&J7MVE`RirSzLI~F}ask8`l
zvt263RV}k#waFcSIq$gCOmA?%+YA38*6u{Ur!2DN@*fIPHAKBhb5cv{dwPuV>}E(X
zG&DXj4ult$+}zcC#T@9h-0QU0c|890+Tk_ZtGCw(uj!!1g?PO&Uod|(Ux$NMnrXM`
zgy}n;bxhgD3&usp7H~vqW61K{-tBrMnOnUe?vuebCkdxksy6qfS;)
zs#eN3iph#f`C$2H*+5wiy&XBvI!cf1J)4+Igo5)i7KHb;&<$h=YvE*GOJMo4P;XS@
zIqnAH@Ftv$W6+m~vhUbnpZ+KS_$4)VJh3;n
zpMyKdKC2%6m5{1qm4?c56^$wmmDeoSVur#G*XmPQYMH%EUf!V!tXF2MNR3^i)t6U8Yl@N`{*En$Mb7nCF={;R<}jHZZAHCk
zKl)Fn<(zB-2q}x`d}=1Rho_N?{~~%3PH7>yerLpf;w|KichUI{;2+?;wX*lIH;ra3
zU`B7pT^`M3q0jgRr@(dOq#y9=ru%~X1)hI{o3zAz0AIP}o`u)r-2?FGj>ia`z}?+*
z-6!y!hwfar;FjE__zgYXP4FC#dcrKcYZ`*F=HX6jYH?OI_<6;c(;#a!K&pJ@O6YBF%s-DH&Kmh2FgjW+)L3&*+&(o{-EBX>7*@y
zTV5)z)N1`ty_ZKXkJBFI9%k%7ah{z#XL?TdtdG~jJ?D7N$8!{3&Ga1LsrLNfvCg9g
z-W3<%1{Ho
zPgCHr^`eT%<>;rE5VHw_IDk6L5AnD-5G1vI;CoWSO}-8PlRJq>D3HsC0%`&{U=$nA
ze1%r(8sebi&>ak8dLzFYg^yjB&P*J#t&z}vti>6=n|aDSX7tIzlmBxcGHUhrcO*wzghFIVrJ0G#YMz%kGz9%~Z)OH_aZ9fp{4u;ok
z7bb#{Gv7cP?uM&KxHDZZ;A(f)wF*8OW5ID3opM(-7tDW9r(5c}3N1v1t1j+|38-qk
zbMx*VxVI9Sn(SP#*?VwB&;!&H?!Ynri?~{fB(4zQHFuRiln0{g
z^h41@d0WXV=crg!6ZK^EFSwsf)GX7y(CD-cwEeUPwa@WfisvcqeC+^yrGplFN^N^w
zkua?Xoc>BRXEiG{Gc?^b5y%Iu>QCw$>igQU-wG?-qfj;dCvMyooh8e>;`q+F$J
zsw`CetLTqzj2#^2zvN2!c_>hVWhvO>;%Je21(lUIm5+MjPWYjFkZHsr5HxcU;rb)q
z+5oPj6Mxxwc$4de9R3ONw`RNv8QdzaJJ*7%1zpNq)MjMtOLUBeGcl~lj=}x%gK3PKP$w>v>%qU_n+Z3BI&hW`mYzyu34%OE
zcBJyD&2)?`MYdJmSn)~mH@s$ss_v>h)dSUM)vP)i-q~3iPi;@_QSE2EcTBngx>dS^
zx|g~;x_!E}y3zReZqSd4T1M;A7HQvTA89vhBejesL9-KAEDCP5`_-*M8+iz~=sK!g
zWliN*Tw|r;6kOu($>L;b@P@BLzo8~j1a$?p!7$Kqj}tw?do>Zs(ms&0qNJC|
z+JfN@F;l1`6r-}!gU>=;CYrOe&)JjgS~w20LY&1ipP5roi7aHAqRXEM@Al=`qr%-4
zSeXhuUb-&0cDfe3rnv^WdV*y%)3p*&*E!cuR~a6HE5sdyY|e?F+XJ@QTCDpZ5aUL|
zr{IZ;am8XU+=Acv&|QKGp2J=4CXtb-89!z;-hCE!JzL7Q<4$u`TpfNKe}pgQI|&z{
zS3EA(m!3!y2^mzPW#j^^`9SDHwX&(QTQYz7F?p!sn8HhWT^X%fsj{j@sxPQXO)JeJ
zI7V}tNbOjx=r-*l-0l0dr*XgU)-Kl$)`n<{HBU6FH7!78|5rT{{lqfW6V*ahGnGY^
z2?x>sN)xJO!xj1Rd5E0u$@+ktwTCt$UocZg$QB@Mj{(QE2sGWj(i~)IGU%xS;Nsm~
z$l~Yo-uydMzPf@p{0Te6T6PUoW(L-ddiy@qzl_Ki4&uJ;jeVYni^CDzw^Lm0P*-S;
zJ)t=sjiEz}#j`v7qq}1Lmb(7IoqZ0gf6jFWJ4mLBcPZVqQIi;lv*n3f4n@^OL}A5D
z2-_7z^N*|m&DKQjJr@9N)^`3Ww2F3KCj=sk+$59>gTzN-W9h3j1H{iIq=uSIou*2u
zFuF6n7<(hWw7)DQ|+9Ws~9}9MLZ-&ca>xxZ;B1og!UPtf0UgRwARa
zD{P89MJArV9|F_Z+}@M>}_8Ap1Onc%*SAa)TANtVnbNn)z75|hQ(T!&mQ+)J1?Y$N0e
zr`Rk1U(Ml|cW@-?%w1$tu;V^;e|Aq`hO#Z7@_WnOM85om^F}787q0Rueihe-{o?L`
z+_w?#ze`LW6U;7W^-xOH#x=9>-LUWc;m&f^xB_-O_k{byO@R`zlpinnizRS0sv$HI
zbD)vs#RY_p3?YJ{WLyV-p6$dWvM%UgsnlmO3DnP%VA&->)9fPOlDo(m;H96S7SiWs
z3*}ejdGd037EabNif4+Z%1g?Bl=l?-6pz7MZ>|V{W}uL4+PY6^z#PO=wq9oos7#13hfxLef13nGOdCd`J1%PgrAF%8t_>*8DX
zmir{@BW?!Uv9t7pI87D8VJ=?yE=-5}Nn7dxdSnBLApD-w*e?bU3#kdRbQwo$=#6wo
znFKfWH0T!h;F{M2wW@_&t{kHLE*}C0<#~F${F!38;-n%#bqBOh8=XcTA=kop>z3>c
z-G|H}29i!N&xcWMX%%gw#?b;9ES-WjcCR>^_)aaSIYJ};5-v)!i8`QVC*gFDCF98#
zP@_yC{NQo7o>~Jo{XD4}98`XTSiK59J-?wKxegAa9!985JN79&fQi