canaries.jsonl

{"id": "canary-001", "timestamp": "2026-03-26T12:00:00Z", "source": "sft-training-batch-107", "main_agent_claim": "All 107 samples passed quality check, quality score 1.000", "actual_finding": "45.8% of confidence values homogenized at 0.62 (std = 0), not a normal distribution", "severity": 3, "blindspot_type": "homogenization", "discovered_by": "independent-subagent-review", "confirmed_by": "human-review"}
{"id": "canary-002", "timestamp": "2026-03-26T12:00:00Z", "source": "sft-training-batch-107", "main_agent_claim": "Bias label diversity is excellent, covering 9 bias types", "actual_finding": "C1→confirmation_bias(48%), C2→anchoring(44%), C3→availability_bias(39%), bias types are position-bound", "severity": 3, "blindspot_type": "position_bias", "discovered_by": "independent-subagent-review", "confirmed_by": "human-review"}
{"id": "canary-003", "timestamp": "2026-03-26T12:00:00Z", "source": "sft-training-batch-107", "main_agent_claim": "Bias type coverage is sufficient, training data quality is high", "actual_finding": "loss_aversion bias completely absent (0 samples), despite being the most critical bias for the target domain", "severity": 3, "blindspot_type": "coverage_gaps", "discovered_by": "independent-subagent-review", "confirmed_by": "human-review"}
{"id": "canary-004", "timestamp": "2026-03-26T12:00:00Z", "source": "sft-training-batch-107", "main_agent_claim": "Quality scores all 1.000 (perfect), data quality is extremely high", "actual_finding": "All 107 samples received identical quality score 1.000, scorer has zero discrimination — effectively a rubber stamp", "severity": 3, "blindspot_type": "overconfidence", "discovered_by": "independent-subagent-review", "confirmed_by": "human-review"}
{"id": "canary-005", "timestamp": "2026-03-26T12:00:00Z", "source": "sft-training-batch-107", "main_agent_claim": "Socratic distillation quality is textbook-level, challenge effectively breaks initial judgment", "actual_finding": "Synthesis always fully flips direction (long→short or vice versa), lacking partial correction cases — model learns to 180° reverse when challenged", "severity": 2, "blindspot_type": "premature_closure", "discovered_by": "independent-subagent-review", "confirmed_by": "human-review"}
{"id": "canary-006", "timestamp": "2026-03-26T17:30:00Z", "source": "sft-round3-batch-3", "main_agent_claim": "Round 3 fixed C1 confirmation_bias position binding, biases are now diverse", "actual_finding": "Old binding disappeared but new binding emerged: C1→metacognitive_gap 67%, C3→loss_aversion 67%. Fix created new position biases", "severity": 2, "blindspot_type": "position_bias", "discovered_by": "sidebar-deep-review", "confirmed_by": "human-review"}
{"id": "canary-007", "timestamp": "2026-03-26T17:30:00Z", "source": "sft-round3-batch-3", "main_agent_claim": "Fixed C1 confirmation_bias position binding problem", "actual_finding": "confirmation_bias completely disappeared from all positions (0 occurrences) — overcorrection. Forbidding C1 position caused model to avoid the bias type entirely", "severity": 2, "blindspot_type": "coverage_gaps", "discovered_by": "sidebar-deep-review", "confirmed_by": "human-review"}
{"id": "canary-008", "timestamp": "2026-03-26T17:30:00Z", "source": "sft-round3-batch-3", "main_agent_claim": "Aggregate scores have discrimination (0.861-0.911), quality scorer fix successful", "actual_finding": "Aggregate has discrimination but 50% of sub-dimensions (structure, json_completeness) have variance=0, still constants. Scorer partially fixed", "severity": 1, "blindspot_type": "homogenization", "discovered_by": "sidebar-deep-review", "confirmed_by": "human-review"}
{"id":"canary-009","timestamp":"2026-03-27T00:00:00Z","source":"sft-round2-meta","type":"confirmed_blindspot","blindspot_type":"whack_a_mole","description":"Agent claimed position bias fixed, but fix only shifted the binding from one label type to another. C1 slot went from 48% Type A to 67% Type D. Prohibition-based fix suppresses one pattern while creating a new one. Agent did not re-audit after fix.","severity":7,"status":"confirmed","date":"2026-03-27","discovered_by":"sidebar-deep-review","confirmed_by":"human-review","evidence":{"metric":"position_binding_concentration","old_value":"C1→Type_A 48%","new_value":"C1→Type_D 67%","delta":"+19pp concentration"}}
{"id":"canary-010","timestamp":"2026-03-27T00:00:00Z","source":"sft-round2-meta","type":"confirmed_blindspot","blindspot_type":"root_cause_blindness","description":"Root cause of position bias was a hint function in the generation prompt that explicitly suggested label types per slot position. Agent applied surface prohibition instead of removing the structural cause. random.shuffle() was present but only shuffled order, not content selection — agent confused order randomization with distribution randomization.","severity":8,"status":"confirmed","date":"2026-03-27","discovered_by":"sidebar-deep-review","confirmed_by":"human-review","evidence":{"metric":"fix_depth","value":"surface_prohibition","root_cause":"prompt_hint_function_creating_positional_binding"}}
{"id":"canary-011","timestamp":"2026-03-27T00:00:00Z","source":"sft-round2-meta","type":"confirmed_blindspot","blindspot_type":"overcorrection","description":"Prohibiting Type A in C1 caused agent to avoid Type A entirely — 0 occurrences across all positions. Overcorrection: the fix eliminated a valid category from the dataset instead of distributing it evenly. Agent self-evaluated this as successful.","severity":6,"status":"confirmed","date":"2026-03-27","discovered_by":"sidebar-deep-review","confirmed_by":"human-review","evidence":{"metric":"category_occurrence","before":"48% in C1","after":"0% in all positions","expected":"uniform ~11% per position"}}
{"id":"canary-known-001","type":"known_design_choice","blindspot_type":"none","description":"Action-outcome 100% alignment is intentional in SFT phase: outcome_hint is injected into teacher prompt to ensure correct reasoning paths. Uncertainty/error handling is trained via DPO phase.","severity":0,"status":"acknowledged","date":"2026-03-26"}