From 002e510fcd4eaa95454a8361ad52d9e7d51376cf Mon Sep 17 00:00:00 2001 From: Stephen Freudenthaler Date: Sat, 13 Jun 2026 11:21:31 -0400 Subject: [PATCH] feat(codex-executor): clear human-facing outcome signaling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewers couldn't always tell a review apart from a non-review. Three gaps: - incomplete/empty and truncated results wore the success header (๐Ÿค– Codex Review) - canceled/timed-out runs left the sticky stuck on "๐Ÿ”„ review in progress" (failure() does not fire on cancellation) - the GPT path is comment-only โ€” nothing surfaced in the PR's checks list Changes: - mantle_review.py emits an outcome flag (ok | truncated | incomplete-empty | error), exported as a step output. - Sticky header now reflects the outcome: ๐Ÿค– Codex Review / โš ๏ธ truncated / โŒ no output โ€” status is visible without opening the body. - New "Fail job when no review was produced" step: after the diagnostic comment is posted, the job exits non-zero on incomplete-empty so a red โœ— shows in PR checks (advisory/non-blocking review, so it surfaces without gating merges). - New "Report cancellation" step (if: cancelled()) rewrites the sticky to "โฑ๏ธ Codex Review canceled" so it never stays stuck on in-progress. No consumer interface change. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/codex-executor.yml | 61 +++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codex-executor.yml b/.github/workflows/codex-executor.yml index 128562b..d264a6c 100644 --- a/.github/workflows/codex-executor.yml +++ b/.github/workflows/codex-executor.yml @@ -339,6 +339,24 @@ jobs: else: f.write("_(model returned no text)_\n") + # Outcome flag consumed by the workflow to pick the comment HEADER and the job exit + # code, so the human-facing status is visible in the sticky header AND in PR checks: + # ok full review -> ๐Ÿค– header, job green + # truncated partial review, hit token cap -> โš ๏ธ header, job green + # incomplete-empty no visible text even after retry -> โŒ header, job RED + # error empty for an undiagnosed reason -> generic failure path (exit 1) + if review and status == "incomplete": + outcome = "truncated" + elif review: + outcome = "ok" + elif status == "incomplete": + outcome = "incomplete-empty" + else: + outcome = "error" + with open("/tmp/outcome.txt", "w", encoding="utf-8") as f: + f.write(outcome) + print(f"outcome: {outcome}", file=sys.stderr) + # Responses API usage: input_tokens, output_tokens, total_tokens, plus # output_tokens_details.reasoning_tokens โ€” the reasoning slice of the output budget. def _g(obj, name, default="?"): @@ -449,25 +467,45 @@ jobs: # vars and the OIDC-issued AWS credentials are inherited by the subprocess. uv run /tmp/mantle_review.py echo "has_review=true" >> "$GITHUB_OUTPUT" + # Surface the outcome (ok | truncated | incomplete-empty) so later steps can pick the + # comment header and decide whether to fail the job. Only reached when the script exits + # 0; a hard error exits 1 before this and is handled by the failure path below. + echo "outcome=$(cat /tmp/outcome.txt 2>/dev/null || echo ok)" >> "$GITHUB_OUTPUT" - name: Update sticky comment with review if: steps.invoke.outputs.has_review == 'true' env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ steps.pr.outputs.number }} + OUTCOME: ${{ steps.invoke.outputs.outcome }} run: | set -euo pipefail USAGE=$(cat /tmp/usage.txt) RUN_LINK="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + # Put the outcome in the HEADER so a human sees the status without reading the body. + case "${OUTCOME:-ok}" in + truncated) HEADER="## โš ๏ธ Codex Review โ€” truncated (partial; hit output-token budget)" ;; + incomplete-empty) HEADER="## โŒ Codex Review โ€” no output (reasoning exhausted the token budget)" ;; + *) HEADER="## ๐Ÿค– Codex Review" ;; + esac { printf "%s\n\n" "${STICKY_MARKER}" - printf "## ๐Ÿค– Codex Review โ€” \`%s\`\n\n" "${MODEL_ID}" + printf "%s โ€” \`%s\`\n\n" "${HEADER}" "${MODEL_ID}" cat /tmp/review.md printf "\n\n---\n" printf "Run: [#%s](%s) ยท tokens: %s\n" "${GITHUB_RUN_ID}" "${RUN_LINK}" "${USAGE}" } > /tmp/comment.md /tmp/sticky-comment.sh "${PR_NUMBER}" /tmp/comment.md + - name: Fail job when no review was produced + # Top-level signal: after the diagnostic comment is posted above, fail the job so the + # "no output" outcome shows as a red โœ— in the PR's checks, not just a buried comment. + # (The review is advisory / non-blocking, so this surfaces it without gating merges.) + if: steps.invoke.outputs.outcome == 'incomplete-empty' + run: | + echo "::error::Codex produced no review โ€” the model exhausted its output-token budget while reasoning, even after a retry. Failing the job so the outcome is visible in checks. Raise max_output_tokens or lower reasoning_effort." + exit 1 + - name: Report failure into sticky comment if: failure() && steps.invoke.outputs.has_review != 'true' env: @@ -483,3 +521,24 @@ jobs: printf "Run: [#%s](%s)\n" "${GITHUB_RUN_ID}" "${RUN_LINK}" } > /tmp/comment.md /tmp/sticky-comment.sh "${PR_NUMBER}" /tmp/comment.md + + - name: Report cancellation into sticky comment + # Cancellation (a newer push superseded this run, or the job timed out) is NOT failure(), + # so without this the sticky would stay stuck on "๐Ÿ”„ review in progress" forever. Runs in + # the cancellation grace period; a single comment update fits well within it. + if: cancelled() + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + run: | + set -euo pipefail + # Only possible once we know the PR; if cancelled before that, there's no sticky yet. + [ -n "${PR_NUMBER:-}" ] || exit 0 + RUN_LINK="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + { + printf "%s\n\n" "${STICKY_MARKER}" + printf "## โฑ๏ธ Codex Review canceled โ€” \`%s\`\n\n" "${MODEL_ID}" + printf "The review run was canceled before completing โ€” superseded by a newer push, or it timed out.\n\n" + printf "Run: [#%s](%s)\n" "${GITHUB_RUN_ID}" "${RUN_LINK}" + } > /tmp/comment.md + /tmp/sticky-comment.sh "${PR_NUMBER}" /tmp/comment.md