Skip to content

Commit 21f7843

Browse files
DDKingerCopilot
andcommitted
Cap consecutive WT listener restart attempts
Stop retrying `wtcli listen` after eight consecutive unstable failures. This keeps a permanently broken COM registration from turning every master/helper into a process and log storm while preserving the agreed degradation contract: chat, Autofix, session list and resume stay available; only live session status remains stale until that WTA process restarts. A subscription that stays healthy for 30 seconds resets the failure streak. The first exit after a stable subscription still restarts immediately because COM broadcasts are not replayed; repeated fast subscribe/exit cycles retain the streak, enter exponential backoff, and eventually stop rather than tight-loop. Added a focused policy test covering the eight-attempt ceiling, fast-exit counting and stable-uptime reset. Validation: 1883 WTA tests passed, full solution built with 0 errors, the SessionHookRouting ItE2E suite passed 3/3 after deployment, and settings.json was unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: c6ef3e46-71af-4e5a-82af-a172700ca3ce
1 parent 6c0c9ed commit 21f7843

1 file changed

Lines changed: 88 additions & 8 deletions

File tree

tools/wta/src/shell/wt_channel/cli_channel.rs

Lines changed: 88 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ const WTCLI_LISTENER_RETRY_INITIAL: Duration = Duration::from_millis(250);
1414
const WTCLI_LISTENER_RETRY_MAX: Duration = Duration::from_secs(5);
1515
const WTCLI_LISTENER_READY_TIMEOUT: Duration = Duration::from_secs(15);
1616
const WTCLI_LISTENER_STDERR_MAX: usize = 16 * 1024;
17+
const WTCLI_LISTENER_MAX_CONSECUTIVE_FAILURES: u32 = 8;
18+
const WTCLI_LISTENER_STABLE_UPTIME: Duration = Duration::from_secs(30);
1719

1820
#[derive(Debug)]
1921
enum WtcliOneShotError {
@@ -208,6 +210,18 @@ fn is_listener_ready_marker(value: &serde_json::Value, token: &str) -> bool {
208210
&& value.get("token").and_then(|value| value.as_str()) == Some(token)
209211
}
210212

213+
fn next_listener_failure_count(current: u32, subscribed: bool, uptime: Duration) -> u32 {
214+
if subscribed && uptime >= WTCLI_LISTENER_STABLE_UPTIME {
215+
1
216+
} else {
217+
current.saturating_add(1)
218+
}
219+
}
220+
221+
fn listener_retry_exhausted(consecutive_failures: u32) -> bool {
222+
consecutive_failures >= WTCLI_LISTENER_MAX_CONSECUTIVE_FAILURES
223+
}
224+
211225
/// Resolve the full path to `wtcli.exe` at startup.
212226
pub(crate) fn resolve_wtcli_path() -> String {
213227
// 1. Explicit override via environment variable.
@@ -595,8 +609,10 @@ impl CliChannel {
595609
/// The protocol server can be temporarily unavailable while Terminal is
596610
/// still starting. `wtcli listen` exits immediately in that window (for
597611
/// example with `E_NOINTERFACE`); a one-shot reader then leaves master
598-
/// permanently blind to hooks and pane lifecycle events. Keep restarting
599-
/// until this channel is dropped or a replacement reader shuts us down.
612+
/// permanently blind to hooks and pane lifecycle events. Retry transient
613+
/// failures, but stop after eight consecutive unstable attempts so a
614+
/// permanently broken COM registration cannot create a process/log storm.
615+
/// A subscription that stays healthy for 30 seconds resets the count.
600616
pub async fn start_reader(self: &std::sync::Arc<Self>) -> bool {
601617
let wtcli = self.wtcli_path.clone();
602618
let weak = std::sync::Arc::downgrade(self);
@@ -611,6 +627,7 @@ impl CliChannel {
611627
let ready_token = format!("wta-{parent_pid}");
612628
let mut ready_tx = Some(ready_tx);
613629
let mut retry_delay = WTCLI_LISTENER_RETRY_INITIAL;
630+
let mut consecutive_failures = 0u32;
614631
loop {
615632
if weak.upgrade().is_none() {
616633
return;
@@ -632,13 +649,28 @@ impl CliChannel {
632649
let mut child = match command.spawn() {
633650
Ok(child) => child,
634651
Err(error) => {
652+
consecutive_failures = next_listener_failure_count(
653+
consecutive_failures,
654+
false,
655+
Duration::ZERO,
656+
);
635657
tracing::warn!(
636658
target: "wtcli",
637659
path = %wtcli,
638660
%error,
661+
consecutive_failures,
662+
max_failures = WTCLI_LISTENER_MAX_CONSECUTIVE_FAILURES,
639663
retry_ms = retry_delay.as_millis(),
640-
"WT protocol event listener spawn failed; retrying"
664+
"WT protocol event listener spawn failed"
641665
);
666+
if listener_retry_exhausted(consecutive_failures) {
667+
tracing::error!(
668+
target: "wtcli",
669+
consecutive_failures,
670+
"WT protocol event listener reached its retry limit; live session status will remain stale until this WTA process restarts"
671+
);
672+
return;
673+
}
642674
tokio::select! {
643675
_ = &mut shutdown_rx => return,
644676
_ = tokio::time::sleep(retry_delay) => {}
@@ -648,6 +680,7 @@ impl CliChannel {
648680
}
649681
};
650682
let listener_pid = child.id();
683+
let listener_started_at = tokio::time::Instant::now();
651684
tracing::info!(
652685
target: "wtcli",
653686
?listener_pid,
@@ -772,17 +805,39 @@ impl CliChannel {
772805
if matches!(exit_reason, "shutdown_requested" | "channel_dropped") {
773806
return;
774807
}
775-
if subscribed {
808+
809+
let stable_subscription =
810+
subscribed && listener_started_at.elapsed() >= WTCLI_LISTENER_STABLE_UPTIME;
811+
if stable_subscription {
776812
retry_delay = WTCLI_LISTENER_RETRY_INITIAL;
813+
}
814+
consecutive_failures = next_listener_failure_count(
815+
consecutive_failures,
816+
subscribed,
817+
listener_started_at.elapsed(),
818+
);
819+
if listener_retry_exhausted(consecutive_failures) {
820+
tracing::error!(
821+
target: "wtcli",
822+
pid = listener_pid,
823+
reason = exit_reason,
824+
consecutive_failures,
825+
"WT protocol event listener reached its retry limit; live session status will remain stale until this WTA process restarts"
826+
);
827+
return;
828+
}
829+
830+
if subscribed && consecutive_failures == 1 {
777831
// The listener had a valid subscription and then died.
778-
// Re-spawn immediately: COM broadcasts are not replayed, so
779-
// an intentional backoff here would create a guaranteed
780-
// hook-loss window. Backoff is reserved for attempts that
781-
// never subscribed successfully.
832+
// Re-spawn the first time immediately: COM broadcasts are
833+
// not replayed. Repeated quick post-subscribe exits retain
834+
// the consecutive-failure count and enter the same backoff
835+
// as pre-subscribe failures, preventing a tight loop.
782836
tracing::warn!(
783837
target: "wtcli",
784838
pid = listener_pid,
785839
reason = exit_reason,
840+
consecutive_failures,
786841
"subscribed WT protocol event listener exited; restarting immediately"
787842
);
788843
continue;
@@ -791,6 +846,8 @@ impl CliChannel {
791846
target: "wtcli",
792847
pid = listener_pid,
793848
reason = exit_reason,
849+
consecutive_failures,
850+
max_failures = WTCLI_LISTENER_MAX_CONSECUTIVE_FAILURES,
794851
retry_ms = retry_delay.as_millis(),
795852
"WT protocol event listener exited; retrying"
796853
);
@@ -1078,6 +1135,29 @@ mod tests {
10781135
));
10791136
}
10801137

1138+
#[test]
1139+
fn listener_retry_limit_counts_only_consecutive_unstable_failures() {
1140+
let mut failures = 0;
1141+
for expected in 1..WTCLI_LISTENER_MAX_CONSECUTIVE_FAILURES {
1142+
failures = next_listener_failure_count(failures, false, Duration::ZERO);
1143+
assert_eq!(failures, expected);
1144+
assert!(!listener_retry_exhausted(failures));
1145+
}
1146+
failures = next_listener_failure_count(failures, true, Duration::from_secs(1));
1147+
assert!(listener_retry_exhausted(failures));
1148+
1149+
// A subscription that stayed healthy long enough starts a new failure
1150+
// streak. A fast subscribe/exit loop deliberately does not.
1151+
assert_eq!(
1152+
next_listener_failure_count(
1153+
WTCLI_LISTENER_MAX_CONSECUTIVE_FAILURES - 1,
1154+
true,
1155+
WTCLI_LISTENER_STABLE_UPTIME
1156+
),
1157+
1
1158+
);
1159+
}
1160+
10811161
#[tokio::test]
10821162
async fn bounded_pipe_reader_keeps_a_prefix_and_drains_the_tail() {
10831163
use tokio::io::AsyncWriteExt;

0 commit comments

Comments
 (0)