Commit 18fea207 by PLN (Algolia)

fix(doctor): churn fails on a storm in progress, warns on one that is over

parent ed2515b9
......@@ -568,27 +568,38 @@ def check_wireplumber_churn() -> None:
The existing checks only see the WRECKAGE (a failed unit, a dummy sink).
This one sees the CAUSE, while audio still works.
"""
def starts_since(window: str) -> int | None:
rc, out, _ = run_cmd(["journalctl", "--user", "-u", "wireplumber",
"--since", "-1h", "--no-pager", "-o", "cat"], timeout=15)
if rc != 0:
"--since", window, "--no-pager", "-o", "cat"], timeout=15)
return None if rc != 0 else len(re.findall(r"^Started wireplumber", out, re.M))
# TWO windows on purpose. A storm an hour ago is history — the box may have
# been healed since, and a check that keeps failing over a fixed past is a
# check PLN learns to scroll past. What blocks a set is a storm that is
# still running.
recent, hour = starts_since("-15min"), starts_since("-1h")
if recent is None or hour is None:
add("DEVICES", "wireplumber churn", WARN, "could not read the journal")
return
starts = len(re.findall(r"^Started wireplumber", out, re.M))
starts = hour
# `bt-audio-fix` is the only known restarter on this box; naming it saves
# the next hour of wondering who is doing this.
rc2, bt, _ = run_cmd(["journalctl", "--user", "--since", "-1h",
"--no-pager", "-o", "cat", "-t", "bt-audio-fix"], timeout=15)
heals = len(re.findall(r"healing WirePlumber|restarting WirePlumber", bt))
detail = f"{starts} start(s) in the last hour" + (f", {heals} from bt-audio-fix" if heals else "")
if starts >= 10:
add("DEVICES", "wireplumber churn", FAIL, detail + " — a restart storm; "
"this ends in start-limit-hit (no devices at all) and it kills every "
"SuperDirt link on the way",
"journalctl --user -t bt-audio-fix --since -1h # who is restarting it, and why")
detail = (f"{recent} start(s) in the last 15 min, {starts} in the hour"
+ (f", {heals} from bt-audio-fix" if heals else ""))
fix = "journalctl --user -t bt-audio-fix --since -1h # who is restarting it, and why"
if recent >= 4:
add("DEVICES", "wireplumber churn", FAIL, detail + " — a storm IN PROGRESS; "
"it ends in start-limit-hit (no devices at all) and it kills every "
"SuperDirt link on the way", fix)
elif starts >= 8:
add("DEVICES", "wireplumber churn", WARN, detail + " — a storm earlier this "
"hour, quiet now; a flapping Bluetooth device is the usual cause", fix)
elif starts >= 4:
add("DEVICES", "wireplumber churn", WARN, detail + " — more than a boot's "
"worth; a flapping Bluetooth device is the usual cause",
"journalctl --user -t bt-audio-fix --since -1h")
"worth of restarts", fix)
else:
add("DEVICES", "wireplumber churn", PASS, detail)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment