feat: judge tier floor and report-level behavioral attribution

Two trust-boundary fixes for the review judge:

- A judge may raise the tier freely, but can no longer fully
  green-light a deterministic :block: when the heuristic tier is
  :block and the judge says :auto, Judge.verdict/2 floors the result
  to :review and records the raw tier in :judge_tier (printed by the
  mix task). Rendered page content feeds the LLM prompt, so :auto —
  "nobody looks at this" — is not a downgrade a model may make alone.

- Run-level behavioral findings (from --timeline) are no longer
  appended to every changed view: one unrelated N+1 no longer marks
  every view :block and double-counts the summary. They stay at the
  report level — printed in their own section and still failing the
  run via the exit gate — and reach the LLM judge as prompt context
  (:run_behavioral) rather than tier arithmetic. The judging
  orchestration moves from the mix task into
  Excessibility.Review.judge_changes/2.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: lessthanseventy

lib/excessibility/review.ex

@@ -21,6 +21,7 @@ defmodule Excessibility.Review do
 
   alias Excessibility.LiveViewRules
   alias Excessibility.Review.Behavioral
+  alias Excessibility.Review.Judge
   alias Excessibility.SnapshotDiff
 
   @type tier :: :auto | :review | :block
@@ -81,6 +82,34 @@ defmodule Excessibility.Review do
     %{changes: changes, summary: summarize(changes)}
   end
 
+  @doc """
+  Run the configured judge over each change in a report.
+
+  Attaches the judge's verdict to every change, replaces the change's
+  tier with the judged tier, and recomputes the summary. Run-level
+  behavioral findings are handed to the judge as context
+  (`:run_behavioral`) but are **not** attributed to any view — they
+  stay at the report level, where `mix excessibility.review` prints
+  them and gates the exit code on them. A view's tier only reflects
+  what that view introduced.
+  """
+  @spec judge_changes(report(), keyword()) :: report()
+  def judge_changes(report, opts \\ []) do
+    behavioral = Map.get(report, :behavioral, [])
+    judge_opts = Keyword.put(opts, :run_behavioral, behavioral)
+
+    judged =
+      Enum.map(report.changes, fn change ->
+        verdict = Judge.verdict(change, judge_opts)
+
+        change
+        |> Map.put(:verdict, verdict)
+        |> Map.put(:tier, verdict.tier)
+      end)
+
+    %{report | changes: judged, summary: summarize(judged)}
+  end
+
   @doc """
   Review a single view's baseline-vs-current snapshot pair.
   """

lib/excessibility/review/judge.ex

@@ -31,11 +31,12 @@ defmodule Excessibility.Review.Judge do
   alias Excessibility.Review.Judge.Heuristic
 
   @type verdict :: %{
-          tier: Excessibility.Review.tier(),
-          blast_radius: String.t(),
-          risks: [%{severity: String.t(), area: String.t(), detail: String.t()}],
-          confidence: float() | nil,
-          source: :heuristic | :llm
+          :tier => Excessibility.Review.tier(),
+          :blast_radius => String.t(),
+          :risks => [%{severity: String.t(), area: String.t(), detail: String.t()}],
+          :confidence => float() | nil,
+          :source => :heuristic | :llm,
+          optional(:judge_tier) => Excessibility.Review.tier()
         }
 
   @callback judge(change :: Excessibility.Review.change(), opts :: keyword()) :: verdict()
@@ -45,13 +46,34 @@ defmodule Excessibility.Review.Judge do
 
   Resolution order: `opts[:judge]`, then `config :excessibility, :review_judge`,
   then `Heuristic`.
+
+  ## Tier floor
+
+  A judge may raise the tier freely, but may not fully green-light a
+  deterministic `:block`: when the change's heuristic tier is `:block`
+  (new critical/serious findings from the rules engine) and the judge
+  says `:auto`, the verdict is floored to `:review` and the judge's raw
+  tier is kept in `:judge_tier`. `:auto` means nobody looks at the
+  change — rendered page content feeds LLM prompts, so that is not a
+  downgrade a model is allowed to make on its own. `:review` still cuts
+  block-level false-positive friction while keeping a human in the loop.
   """
   @spec verdict(Excessibility.Review.change(), keyword()) :: verdict()
   def verdict(change, opts \\ []) do
     judge =
       Keyword.get(opts, :judge) ||
         Application.get_env(:excessibility, :review_judge, Heuristic)
 
-    judge.judge(change, opts)
+    change
+    |> judge.judge(opts)
+    |> apply_tier_floor(change)
   end
+
+  defp apply_tier_floor(%{tier: :auto} = verdict, %{tier: :block}) do
+    verdict
+    |> Map.put(:tier, :review)
+    |> Map.put(:judge_tier, :auto)
+  end
+
+  defp apply_tier_floor(verdict, _change), do: verdict
 end

lib/excessibility/review/judge/llm.ex

@@ -44,7 +44,7 @@ defmodule Excessibility.Review.Judge.LLM do
   end
 
   defp run(change, fun, opts) do
-    with {:ok, text} when is_binary(text) <- safe_call(fun, build_prompt(change)),
+    with {:ok, text} when is_binary(text) <- safe_call(fun, build_prompt(change, opts)),
          {:ok, verdict} <- parse(text, change) do
       verdict
     else
@@ -92,7 +92,7 @@ defmodule Excessibility.Review.Judge.LLM do
 
   defp parse_risks(_), do: []
 
-  defp build_prompt(change) do
+  defp build_prompt(change, opts) do
     """
     You are a release-risk judge for a Phoenix/LiveView change. Decide whether
     this view's change is safe to auto-merge given a strong test + painless
@@ -116,6 +116,9 @@ defmodule Excessibility.Review.Judge.LLM do
 
     Behavioral findings (from telemetry analyzers — queries, state, renders):
     #{render_behavioral(Map.get(change, :behavioral, []))}
+
+    Run-level behavioral findings (whole test run — context only, not attributed to this view):
+    #{render_behavioral(Keyword.get(opts, :run_behavioral, []))}
     """
   end
 

lib/mix/tasks/excessibility_review.ex

@@ -34,7 +34,6 @@ defmodule Mix.Tasks.Excessibility.Review do
   use Mix.Task
 
   alias Excessibility.Review
-  alias Excessibility.Review.Judge
 
   @requirements ["app.config"]
 
@@ -46,7 +45,7 @@ defmodule Mix.Tasks.Excessibility.Review do
     fail_on = parse_fail_on(opts[:fail_on])
 
     report = Review.review(review_opts(opts))
-    report = if Keyword.get(opts, :judge, false), do: judge_report(report), else: report
+    report = if Keyword.get(opts, :judge, false), do: Review.judge_changes(report), else: report
 
     print_report(report)
     maybe_exit(report, fail_on)
@@ -62,30 +61,6 @@ defmodule Mix.Tasks.Excessibility.Review do
     end
   end
 
-  # Run the configured judge over each change. The run-level behavioral
-  # findings are attached to every change so the judge weighs behavior
-  # alongside markup; the verdict's tier overrides the heuristic one.
-  defp judge_report(report) do
-    behavioral = Map.get(report, :behavioral, [])
-
-    judged =
-      Enum.map(report.changes, fn change ->
-        change = Map.update(change, :behavioral, behavioral, &(&1 ++ behavioral))
-        verdict = Judge.verdict(change)
-        change |> Map.put(:verdict, verdict) |> Map.put(:tier, verdict.tier)
-      end)
-
-    counts = Enum.frequencies_by(judged, & &1.tier)
-
-    summary = %{
-      auto: Map.get(counts, :auto, 0),
-      review: Map.get(counts, :review, 0),
-      block: Map.get(counts, :block, 0)
-    }
-
-    %{changes: judged, behavioral: behavioral, summary: summary}
-  end
-
   defp parse_fail_on(nil), do: :block
   defp parse_fail_on("block"), do: :block
   defp parse_fail_on("review"), do: :review
@@ -149,6 +124,13 @@ defmodule Mix.Tasks.Excessibility.Review do
     Enum.each(verdict.risks, fn risk ->
       Mix.shell().info("    - [#{risk.severity}] #{risk.area}: #{risk.detail}")
     end)
+
+    if judge_tier = Map.get(verdict, :judge_tier) do
+      Mix.shell().info(
+        "    (judge said #{judge_tier}; floored to #{verdict.tier} — " <>
+          "a judge cannot fully green-light new serious findings)"
+      )
+    end
   end
 
   defp tier_rank(:block), do: 0

test/review/judge_test.exs

@@ -133,4 +133,69 @@ defmodule Excessibility.Review.JudgeTest do
                :heuristic
     end
   end
+
+  describe "verdict/2 tier floor" do
+    defp auto_change do
+      Review.review_pair("home", "<div>hi</div>", "<div>hi</div>")
+    end
+
+    defp completion_replying(tier) do
+      fn _prompt ->
+        {:ok, ~s({"tier":"#{tier}","blast_radius":"model says so","risks":[],"confidence":0.9})}
+      end
+    end
+
+    test "floors a model's :auto verdict to :review when the heuristic says :block" do
+      verdict = Judge.verdict(block_change(), judge: LLM, completion: completion_replying("auto"))
+
+      assert verdict.tier == :review
+      assert verdict.judge_tier == :auto
+      assert verdict.source == :llm
+    end
+
+    test "allows a model to downgrade :block to :review" do
+      verdict = Judge.verdict(block_change(), judge: LLM, completion: completion_replying("review"))
+
+      assert verdict.tier == :review
+      refute Map.has_key?(verdict, :judge_tier)
+    end
+
+    test "allows a judge to raise the tier without restriction" do
+      verdict = Judge.verdict(auto_change(), judge: LLM, completion: completion_replying("block"))
+
+      assert verdict.tier == :block
+    end
+
+    test "a model's :auto verdict on an :auto change is untouched" do
+      verdict = Judge.verdict(auto_change(), judge: LLM, completion: completion_replying("auto"))
+
+      assert verdict.tier == :auto
+      refute Map.has_key?(verdict, :judge_tier)
+    end
+  end
+
+  describe "run-level behavioral context" do
+    test "run_behavioral findings are rendered into the LLM prompt as context" do
+      completion = fn prompt ->
+        send(self(), {:prompt, prompt})
+        {:ok, ~s({"tier":"review","blast_radius":"x","risks":[],"confidence":0.5})}
+      end
+
+      behavioral = [
+        %{
+          severity: :serious,
+          rule: :ecto_query_analysis,
+          message: "N+1 in orders",
+          source: :telemetry,
+          events: []
+        }
+      ]
+
+      Judge.verdict(block_change(), judge: LLM, completion: completion, run_behavioral: behavioral)
+
+      assert_received {:prompt, prompt}
+      assert prompt =~ "N+1 in orders"
+      assert prompt =~ "not attributed"
+    end
+  end
 end

test/review_test.exs

@@ -113,4 +113,33 @@ defmodule Excessibility.ReviewTest do
       assert [%{rule: :stub}] = report.behavioral
     end
   end
+
+  describe "judge_changes/2" do
+    test "attaches verdicts and recomputes the summary from judged tiers" do
+      report = Review.review_pairs([{"editor", "<div>Save</div>", ~s(<div phx-click="save">Save</div>)}])
+
+      judged = Review.judge_changes(report)
+
+      assert [%{verdict: %{source: :heuristic}, tier: :block}] = judged.changes
+      assert judged.summary.block == 1
+    end
+
+    test "run-level behavioral findings do not escalate per-view tiers" do
+      # An :auto view change plus an unrelated run-level critical finding:
+      # the run-level finding must not mark this view :block.
+      report =
+        [{"search", ~s(<div aria-live="polite">0</div>), ~s(<div aria-live="polite">1</div>)}]
+        |> Review.review_pairs()
+        |> Map.put(:behavioral, [
+          %{severity: :serious, rule: :stub, message: "N+1 in orders", source: :telemetry, events: []}
+        ])
+
+      judged = Review.judge_changes(report)
+
+      assert [%{tier: :auto}] = judged.changes
+      assert judged.summary == %{auto: 1, review: 0, block: 0}
+      # ...but they are preserved at the report level for the exit gate.
+      assert [%{message: "N+1 in orders"}] = judged.behavioral
+    end
+  end
 end