feat(complexity_router): bound the classifier context block, not each turn in it (#38145)

The LLM classifier capped every prior turn at 200 characters independently, so a
785 character turn was cut even when the whole block it belonged to was 353
characters. A character budget now bounds the block: turns are taken newest first
and quoted whole while they fit, older turns are dropped whole once it runs out,
and only the turn straddling the boundary is cut. The per-turn cap stays as an
optional clamp for operators who set it deliberately, defaulting to unset.

Author: tin-berri

litellm/router_strategy/complexity_router/complexity_router.py

@@ -19,7 +19,7 @@
 import random
 import re
 from collections.abc import Iterator, Mapping, Sequence
-from itertools import accumulate, islice
+from itertools import accumulate, islice, takewhile
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
 
@@ -275,6 +275,7 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, Any] | None
 
 _TRUNCATION_MARKER: Final = "..."
 _TRUNCATION_HEAD_FRACTION: Final = 0.3
+_MIN_QUOTED_TURN_CHARS: Final = 120
 
 _CJK_CHARACTER: Final = re.compile("[぀-ヿㇰ-ㇿ㐀-䶿一-鿿豈-﫿ｦ-ﾝ\U00020000-\U0003ffff]")
 
@@ -593,11 +594,40 @@ def _iter_context_turns_newest_first(
     )
 
 
+def _turns_within_budget(
+    turns: Sequence[tuple[str, str]],
+    budget_chars: int,
+) -> tuple[tuple[str, str], ...]:
+    """The newest-first turns that fit budget_chars, quoted whole wherever they fit.
+
+    Bounding the block rather than every turn in it is what lets an ordinary conversation reach the
+    classifier intact: a per-turn cap cuts a 785 character turn even when the whole block would have
+    been 353 characters, which is three orders of magnitude below anything the classifier call is
+    near. Once the budget does run out the older turns are dropped entire rather than shortened, so
+    at most one turn is ever cut and the rest read as themselves. A remainder too small to carry a
+    sentence buys less signal than the ellipses it would arrive wrapped in, so that turn is dropped.
+
+    The boundary turn is cut to leave room for the marker rather than to the remainder itself, so the
+    quoted block never exceeds budget_chars; the marker is part of what the budget buys, not an extra
+    charged on top of it.
+    """
+    spent: Final = accumulate(len(text) for _, text in turns)
+    fitting: Final = tuple(takewhile(lambda pair: pair[1] <= budget_chars, zip(turns, spent)))
+    remaining: Final = budget_chars - (fitting[-1][1] if fitting else 0)
+    whole: Final = tuple(turn for turn, _ in fitting)
+    cut_to: Final = remaining - len(_TRUNCATION_MARKER)
+    if len(whole) == len(turns) or cut_to < _MIN_QUOTED_TURN_CHARS:
+        return whole
+    boundary_role, boundary_text = turns[len(whole)]
+    return (*whole, (boundary_role, _truncate(boundary_text, cut_to)))
+
+
 def _extract_prior_turns(
     messages: Sequence[Mapping[str, object]],
     current_ask: str | None,
     window_size: int,
-    per_turn_chars: int,
+    budget_chars: int,
+    per_turn_chars: int | None,
     include_assistant: bool,
     marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
 ) -> tuple[tuple[str, str], ...]:
@@ -612,19 +642,29 @@ def _extract_prior_turns(
     window_size counts turns of every eligible role, so with assistant turns included it is the last N
     of the conversation rather than the last N asks. A turn carrying only tool calls or thinking
     blocks flattens to empty text and is skipped, so it never spends a slot.
+
+    Three bounds apply and the tightest wins: window_size caps how many turns, budget_chars caps the
+    block they form, and per_turn_chars optionally caps any single one of them before the block is
+    measured. They are separate because they answer separate questions, and only the block bound
+    tracks what the classifier call actually costs.
     """
     if window_size <= 0 or not messages:
         return ()
 
-    prior: Final = islice(
-        (
-            turn
-            for turn in _iter_context_turns_newest_first(messages, include_assistant, marker_pairs)
-            if turn[1] != current_ask
-        ),
-        window_size,
+    prior: Final = tuple(
+        islice(
+            (
+                turn
+                for turn in _iter_context_turns_newest_first(messages, include_assistant, marker_pairs)
+                if turn[1] != current_ask
+            ),
+            window_size,
+        )
+    )
+    clamped: Final = (
+        prior if per_turn_chars is None else tuple((role, _truncate(text, per_turn_chars)) for role, text in prior)
     )
-    return tuple((role, _truncate(text, per_turn_chars)) for role, text in reversed(tuple(prior)))
+    return tuple(reversed(_turns_within_budget(clamped, budget_chars)))
 
 
 def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bool:
@@ -1363,6 +1403,7 @@ async def _classify_with_llm(
                 messages,
                 current_ask=prompt,
                 window_size=self.config.classifier_context_window_size,
+                budget_chars=self.config.classifier_context_budget_chars,
                 per_turn_chars=self.config.classifier_context_per_turn_chars,
                 include_assistant=include_assistant,
                 marker_pairs=self._reminder_markers,

litellm/router_strategy/complexity_router/config.py

@@ -49,7 +49,7 @@ class ClassificationRubric(str, Enum):
 DEFAULT_TIER_DISTANCE_PENALTY: Final[float] = 0.5
 
 DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE: Final[int] = 3
-DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS: Final[int] = 200
+DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS: Final[int] = 8000
 
 
 class KeywordTierRule(BaseModel):
@@ -645,12 +645,30 @@ class ComplexityRouterConfig(BaseModel):
             "classifier_type is 'llm'."
         ),
     )
-    classifier_context_per_turn_chars: int = Field(
-        default=DEFAULT_CLASSIFIER_CONTEXT_PER_TURN_CHARS,
+    classifier_context_budget_chars: int = Field(
+        default=DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS,
+        ge=0,
+        description=(
+            "Maximum characters of prior-turn text quoted to the LLM classifier, across the whole "
+            "context window, per classification call. Turns are taken newest first and quoted whole "
+            "while they fit, so a conversation small enough to quote entirely is never cut; once the "
+            "budget runs out the older turns are dropped whole and only the turn straddling the "
+            "boundary is truncated, into whatever space is left. The current ask and the caller's "
+            "system prompt sit outside this budget and are always sent in full, as does the numbering "
+            "each quoted turn carries. A budget under 120 leaves no room to quote a turn and "
+            "suppresses the block; set classifier_context_window_size to 0 to turn context off "
+            "deliberately. Only applies when classifier_type is 'llm'."
+        ),
+    )
+    classifier_context_per_turn_chars: int | None = Field(
+        default=None,
         gt=0,
         description=(
-            "Maximum character length for each prior turn's text in the classifier context window. "
-            "Turns exceeding this are truncated. Only applies when classifier_type is 'llm'."
+            "Optional cap on each individual prior turn's text, applied before "
+            "classifier_context_budget_chars bounds the block. Unset by default, so one long turn may "
+            "spend the whole budget, which is usually what a follow-up needs; set it when no single "
+            "turn should dominate the context the classifier sees. A capped turn keeps its opening "
+            "and its ending with the middle elided. Only applies when classifier_type is 'llm'."
         ),
     )
     classifier_context_include_assistant_turns: bool = Field(
@@ -662,9 +680,9 @@ class ComplexityRouterConfig(BaseModel):
             "word 'yes'. When enabled, classifier_context_window_size counts the last N turns of the "
             "conversation across both roles rather than the last N user turns, and assistant text is "
             "sent to the classifier model, which may be a different deployment or provider than the "
-            "routed completion model. Assistant replies share classifier_context_per_turn_chars with "
-            "user turns, so raise it if replies are truncated before the part that carries the "
-            "difficulty. Off by default because enabling it shifts tier decisions, and therefore "
+            "routed completion model. Assistant replies spend classifier_context_budget_chars "
+            "alongside user turns, so raise it if the oldest turns stop being quoted once replies "
+            "join the window. Off by default because enabling it shifts tier decisions, and therefore "
             "spend, for an already-deployed router. Only applies when classifier_type is 'llm'."
         ),
     )

tests/test_litellm/router_strategy/test_complexity_router.py

@@ -6052,8 +6052,9 @@ def test_clipped_prior_turn_still_carries_the_ask_it_closes_on(self):
             [{"role": "user", "content": turn}, {"role": "user", "content": "go ahead"}],
             "go ahead",
             3,
-            200,
-            False,
+            budget_chars=10_000,
+            per_turn_chars=200,
+            include_assistant=False,
         )
 
         assert "multi-region gateway" in quoted[0][1]
@@ -6216,7 +6217,167 @@ def test_prior_turn_window(self, messages, current_ask, window, per_turn_chars,
         """
         from litellm.router_strategy.complexity_router.complexity_router import _extract_prior_turns
 
-        assert _extract_prior_turns(messages, current_ask, window, per_turn_chars, include_assistant) == expected
+        assert (
+            _extract_prior_turns(
+                messages,
+                current_ask,
+                window,
+                budget_chars=10_000,
+                per_turn_chars=per_turn_chars,
+                include_assistant=include_assistant,
+            )
+            == expected
+        )
+
+    @pytest.mark.parametrize(
+        "turn_lengths,budget_chars,expected_lengths",
+        [
+            pytest.param((50, 50, 50), 10_000, (50, 50, 50), id="a-block-that-fits-is-quoted-whole"),
+            pytest.param((100, 100, 100), 250, (100, 100), id="oldest-turn-is-dropped-whole"),
+            pytest.param((500, 100), 400, (300, 100), id="only-the-boundary-turn-is-cut"),
+            pytest.param((900,), 300, (300,), id="a-turn-larger-than-the-budget-is-still-quoted"),
+            pytest.param((500, 100), 180, (100,), id="a-remainder-too-small-to-carry-a-sentence-is-dropped"),
+            pytest.param((50,), 0, (), id="a-zero-budget-quotes-nothing"),
+        ],
+    )
+    def test_budget_bounds_the_block_not_each_turn(self, turn_lengths, budget_chars, expected_lengths):
+        """Turns are taken newest first and quoted whole while they fit.
+
+        The defect this replaces capped every turn independently, so a 785 character turn was cut even
+        though the whole block it belonged to was 353 characters. Bounding the block instead means an
+        ordinary conversation arrives intact, and when the budget really does run out the older turns
+        are dropped entire rather than each arriving mangled. At most one turn is ever cut, and a
+        remainder too small to carry a sentence is dropped rather than quoted as two ellipses around a
+        fragment. A single turn bigger than the whole budget is still quoted, cut to the budget, since
+        dropping it would leave the classifier with no context at all.
+        """
+        from litellm.router_strategy.complexity_router.complexity_router import _extract_prior_turns
+
+        messages = [{"role": "user", "content": f"{i}" * length} for i, length in enumerate(turn_lengths)]
+
+        quoted = _extract_prior_turns(
+            [*messages, {"role": "user", "content": "go ahead"}],
+            "go ahead",
+            len(turn_lengths),
+            budget_chars=budget_chars,
+            per_turn_chars=None,
+            include_assistant=False,
+        )
+
+        assert tuple(len(text) for _, text in quoted) == expected_lengths
+
+    @pytest.mark.parametrize("budget_chars", [130, 200, 351, 400, 999, 8000])
+    @pytest.mark.parametrize("turn_lengths", [(900,), (500, 100), (100, 100, 100), (50, 50, 50)])
+    def test_the_quoted_block_never_exceeds_the_budget(self, turn_lengths, budget_chars):
+        """The budget is a ceiling on what is quoted, marker included.
+
+        Cutting the boundary turn to the remainder and then appending the marker put the block three
+        characters over the number an operator configured, which is the kind of drift that makes a
+        documented ceiling untrue. Asserted across shapes rather than at the one boundary that happened
+        to be wrong, so any future off-by-marker anywhere in the fill is caught here.
+        """
+        from litellm.router_strategy.complexity_router.complexity_router import _extract_prior_turns
+
+        messages = [{"role": "user", "content": f"{i}" * length} for i, length in enumerate(turn_lengths)]
+
+        quoted = _extract_prior_turns(
+            [*messages, {"role": "user", "content": "go ahead"}],
+            "go ahead",
+            len(turn_lengths),
+            budget_chars=budget_chars,
+            per_turn_chars=None,
+            include_assistant=False,
+        )
+
+        assert sum(len(text) for _, text in quoted) <= budget_chars
+
+    def test_per_turn_cap_still_clamps_when_an_operator_sets_it(self):
+        """An operator who set the per-turn cap keeps exactly what they configured.
+
+        The cap stopped being the default, so it has to keep working for the deployments that named it
+        deliberately; it applies before the block budget rather than instead of it.
+        """
+        from litellm.router_strategy.complexity_router.complexity_router import _extract_prior_turns
+
+        quoted = _extract_prior_turns(
+            [{"role": "user", "content": "z" * 900}, {"role": "user", "content": "go ahead"}],
+            "go ahead",
+            3,
+            budget_chars=10_000,
+            per_turn_chars=200,
+            include_assistant=False,
+        )
+
+        assert len(quoted[0][1]) == 203
+
+    @pytest.mark.asyncio
+    async def test_a_long_turn_reaches_the_classifier_whole_by_default(
+        self, mock_router_instance, llm_classifier_config
+    ):
+        """The shipped defaults quote an ordinary long turn without cutting it anywhere.
+
+        This is the whole point of the change, asserted where a deployment actually meets it: no knob
+        set, one turn well past the retired 200 character cap, and no truncation marker in the payload.
+        """
+        from litellm.router_strategy.complexity_router.complexity_router import _TRUNCATION_MARKER
+
+        router = ComplexityRouter(
+            model_name="test-complexity-router",
+            litellm_router_instance=mock_router_instance,
+            complexity_router_config=llm_classifier_config,
+        )
+        mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
+        turn = "The incident ran from 02:10 to 02:40 and only streaming was affected. " * 10 + "Now rewrite it"
+
+        await router.aclassify(
+            "go ahead",
+            messages=[{"role": "user", "content": turn}, {"role": "user", "content": "go ahead"}],
+        )
+
+        user_payload = mock_router_instance.acompletion.call_args.kwargs["messages"][1]["content"]
+        assert turn in user_payload
+        assert _TRUNCATION_MARKER not in user_payload
+
+    @pytest.mark.asyncio
+    async def test_a_turn_dropped_for_budget_still_counts_as_prior_conversation(
+        self, mock_router_instance, llm_classifier_config
+    ):
+        """Dropping turns to fit the budget must not make a long conversation look single-turn.
+
+        The depth line gates on whether prior conversation exists, not on whether any of it was worth
+        quoting, exactly so a continuation is never reported as a context-free first request. A budget
+        tight enough to drop every turn is the newest way to reach that mismatch.
+        """
+        router = ComplexityRouter(
+            model_name="test-complexity-router",
+            litellm_router_instance=mock_router_instance,
+            complexity_router_config={**llm_classifier_config, "classifier_context_budget_chars": 1},
+        )
+        mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
+
+        await router.aclassify(
+            "go ahead",
+            messages=[
+                {"role": "user", "content": "a long earlier request that cannot fit a one character budget"},
+                {"role": "user", "content": "go ahead"},
+            ],
+        )
+
+        user_payload = mock_router_instance.acompletion.call_args.kwargs["messages"][1]["content"]
+        assert "Recent conversation" not in user_payload
+        assert "Conversation so far" in user_payload
+
+    def test_context_defaults_bound_the_block_and_leave_turns_uncapped(self):
+        """The shipped defaults: a block budget, and no per-turn cap unless one is named."""
+        from litellm.router_strategy.complexity_router.config import (
+            DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS,
+            ComplexityRouterConfig,
+        )
+
+        config = ComplexityRouterConfig()
+
+        assert config.classifier_context_budget_chars == DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS
+        assert config.classifier_context_per_turn_chars is None
 
     def test_prior_turn_context_strips_every_configured_pair(self):
         """The classifier's context window is stripped with the same pairs as the ask.
@@ -6235,7 +6396,7 @@ def test_prior_turn_context_strips_every_configured_pair(self):
             {"role": "user", "content": "current ask"},
         ]
 
-        assert _extract_prior_turns(messages, "current ask", 5, 200, False, pairs) == (
+        assert _extract_prior_turns(messages, "current ask", 5, 10_000, 200, False, pairs) == (
             ("user", "what about b-trees?"),
             ("user", "and heaps?"),
         )