fix(web): retry JS-shell 200s past every cache and name the escalation

crates/tui/src/tools/fetch_url.rs

@@ -14,7 +14,8 @@ use super::spec::{
 };
 use super::web::extract::{DocumentKind, ExtractedDocument, decode_response_body};
 use super::web::fetch::{
-    DEFAULT_MAX_BYTES, DEFAULT_TIMEOUT, FetchOptions, HARD_MAX_BYTES, HARD_MAX_TIMEOUT, fetch,
+    DEFAULT_MAX_BYTES, DEFAULT_TIMEOUT, FetchAttempt, FetchOptions, HARD_MAX_BYTES,
+    HARD_MAX_TIMEOUT, fetch_readable,
 };
 use super::web::overflow::bound_text as bound_web_text;
 #[cfg(test)]
@@ -73,6 +74,10 @@ struct FetchReceipt {
     cache_hit: bool,
     retries: usize,
     redirects: usize,
+    /// Every request this fetch made, in order: session `cache_hit`, whether
+    /// the attempt bypassed caches, which one produced content, and the
+    /// response headers that explain the cache state (#5904).
+    attempts: Vec<FetchAttempt>,
 }
 
 #[derive(Debug)]
@@ -165,38 +170,56 @@ impl ToolSpec for FetchUrlTool {
         let timeout_ms = optional_u64(&input, "timeout_ms", DEFAULT_TIMEOUT.as_millis() as u64)?
             .clamp(1, HARD_MAX_TIMEOUT.as_millis() as u64);
         let requested_fields = parse_fields(&input)?;
-        let fetched = fetch(
+        // Bound as a reference so the `Fn` extractor can run twice without
+        // moving the field list into its first future.
+        let requested_fields = &requested_fields;
+        // A 2xx that extracts to nothing is re-fetched once past every cache
+        // before it becomes an error; the receipt keeps both attempts (#5904).
+        let readable = fetch_readable(
             &url,
             &FetchOptions::new(Duration::from_millis(timeout_ms), max_bytes, FETCH_ACCEPT),
             context,
             "fetch_url",
+            |fetched: super::web::fetch::FetchedPayload| {
+                Box::pin(async move {
+                    let is_success = (200..300).contains(&fetched.status);
+                    let body_text = if requested_fields.is_empty() {
+                        None
+                    } else {
+                        // JSON is never allowed to discover an encoding from body markup.
+                        Some(decode_response_body(
+                            &fetched.bytes,
+                            Some(&fetched.content_type),
+                            false,
+                        )?)
+                    };
+                    let fields = match body_text.as_deref() {
+                        Some(body) => {
+                            project_json_fields(body, &fetched.content_type, requested_fields)?
+                        }
+                        None => None,
+                    };
+                    let extracted = extract_fetched_document(
+                        format,
+                        &fetched.url,
+                        &fetched.content_type,
+                        &fetched.bytes,
+                        is_success,
+                        body_text.as_deref(),
+                        PdfTextCommand::system(context.cancel_token.as_ref()),
+                    )
+                    .await?;
+                    Ok((extracted, fields))
+                })
+            },
         )
         .await?;
+        let super::web::fetch::ReadableFetch {
+            payload: fetched,
+            document: (extracted, fields),
+            attempts,
+        } = readable;
         let is_success = (200..300).contains(&fetched.status);
-        let body_text = if requested_fields.is_empty() {
-            None
-        } else {
-            // JSON is never allowed to discover an encoding from body markup.
-            Some(decode_response_body(
-                &fetched.bytes,
-                Some(&fetched.content_type),
-                false,
-            )?)
-        };
-        let fields = match body_text.as_deref() {
-            Some(body) => project_json_fields(body, &fetched.content_type, &requested_fields)?,
-            None => None,
-        };
-        let extracted = extract_fetched_document(
-            format,
-            &fetched.url,
-            &fetched.content_type,
-            &fetched.bytes,
-            is_success,
-            body_text.as_deref(),
-            PdfTextCommand::system(context.cancel_token.as_ref()),
-        )
-        .await?;
 
         let citation_title = extracted.title.clone();
         let (processed, artifact_write) = render_extracted(
@@ -229,6 +252,7 @@ impl ToolSpec for FetchUrlTool {
                 cache_hit: fetched.cache_hit,
                 retries: fetched.retries,
                 redirects: fetched.redirects,
+                attempts,
             },
             artifact,
             fields,

crates/tui/src/tools/web/extract.rs

@@ -363,9 +363,24 @@ fn markdown_title(body: &str) -> Option<String> {
     })
 }
 
+/// Stable prefix for "the response parsed, but carried no readable body".
+///
+/// The fetch pipeline matches on this to decide whether a cache-busting
+/// re-fetch is worth one more request, and to attach the role-aware recovery
+/// text. Extraction itself has no `ToolContext`, so it cannot know which
+/// escalation the calling role actually owns; it states the fact and leaves
+/// the remedy to [`super::fetch`].
+pub(crate) const JS_SHELL_MARKER: &str = "No readable page content was found at";
+
+/// Whether `error` is the JS-shell extraction failure (a parsed response whose
+/// body held no readable content), as opposed to a transport or type failure.
+pub(crate) fn is_js_shell_error(error: &ToolError) -> bool {
+    error.to_string().contains(JS_SHELL_MARKER)
+}
+
 fn js_required_error(url: &str) -> ToolError {
     ToolError::execution_failed(format!(
-        "No readable page content was found at {url}; the page may require JavaScript. Recovery: use browser automation for this URL."
+        "{JS_SHELL_MARKER} {url}; the response parsed but its body held no readable content, so the page may require JavaScript."
     ))
 }
 
@@ -898,8 +913,19 @@ mod tests {
         .expect_err("empty app shell must fail");
 
         let message = error.to_string();
-        assert!(message.contains("may require JavaScript"));
-        assert!(message.contains("browser automation"));
+        assert!(message.contains("may require JavaScript"), "{message}");
+        assert!(
+            message.contains("https://example.com/app"),
+            "the shell failure must name the URL: {message}"
+        );
+        assert!(
+            is_js_shell_error(&error),
+            "the fetch pipeline recognizes this failure by marker: {message}"
+        );
+        assert!(
+            !is_js_shell_error(&ToolError::execution_failed("connection reset")),
+            "transport failures must not look like a JS shell"
+        );
     }
 
     #[tokio::test]