christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Audio failover: press FOH Backup by itself when Studio B dies; Companion button presses as fixes

Browse Source 
A new fix kind "companion_press" presses any Companion button over its HTTP API (page, row, col).
A fix marked "failover" runs by itself within seconds of its check failing: inside the "failover"
windows (Sunday 06:30-13:30), even while ALARMS is muted, once per fault, never back the other way.
The hidden STB AUDIO key (inside AUDIO) watches Studio B's ping every second, Pro Tools over SSH
every 2 s and its Dante device; each check's fix is the FOH Backup press. The keys say ON FOH BACKUP
until STREAM STB on the Restart page switches back. Settings page: the press builder, the failover
tick and undo, the Failover calendar, help entry "A fallback button".

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-15 12:40:24 -04:00
co-authored by Claude Fable 5.1
parent 422e92f814
commit 14fce2cf9a
13 changed files with 855 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
status_panel.py
+279 -32
View File
@@ -40,9 +40,16 @@ from datetime import datetime
HERE = os.path.dirname(os.path.abspath(__file__))
log = logging.getLogger("status-panel")
VERSION = "2026.10.13" # logged at start ("status panel running (code ...)") and written to started.json
VERSION = "2026.10.15" # logged at start ("status panel running (code ...)") and written to started.json
# Every kind of fix the watchdog can run (validated by load_config).
FIX_KINDS = ("ssh", "local", "atem", "reconnect", "midi_connect", "netgear_poe", "hyperdeck", "spotify", "ready")
FIX_KINDS = ("ssh", "local", "atem", "reconnect", "midi_connect", "netgear_poe", "hyperdeck", "spotify", "ready",
"companion_press")
# Failover: a fix marked "failover" (pressing the FOH Backup button when Studio B dies) runs by itself
# inside these windows when the "failover" block names none, "after_seconds" after its fault is confirmed.
FAILOVER_WINDOWS = [{"days": ["sun"], "start": "06:30", "end": "13:30"}]
REARM_SECONDS = 10 # a failover is armed again once its fault has been gone this long (not on one good reading)
PING_INTERVAL = 3 # seconds between pings; a ping check may ask for less ("interval_seconds")
REMOTE_INTERVAL = 10 # seconds between a remote check's ssh runs; a check may ask for less
OK, WARN, ERR, OFF = "ok", "warn", "err", "off"
@@ -143,6 +150,12 @@ class Companion:
status, _ = self._request("POST", "/api/connections/%s/enable" % urllib.parse.quote(conn_id, safe=""))
return status == 200
def press(self, page, row, col):
"""Press a button (down, then up 20 ms later) and return the HTTP status: 200 when a button is
there, 204 ("No control") when that cell is empty."""
status, _ = self._request("POST", "/api/location/%d/%d/%d/press" % (page, row, col))
return status
# --- Shared snapshot written by the pollers --------------------------------
@@ -538,6 +551,20 @@ def check_remote(chk, snap, ctx):
return [("remote", Result(OK))]
def no_verdict(chk, snap, now):
"""True when a check has nothing to say yet: its ping has not run, ssh could not ask, or Companion's
Dante module has no device list. The failover treats that as "not clear yet"."""
t = chk["type"]
if t == "ping":
return snap.ping.get(ping_target(chk, snap)) is None
if t == "remote":
return snap.remote.get(remote_key(chk)) is None
if t == "dante":
raw = snap.var(chk["conn"], "devices")
return conn_result(snap, chk["conn"], now).level != OK or not raw
return False
def check_ready(chk, snap, ctx):
"""One line of the weekly checklist on the READY page; readiness.py does the looking. Until the
checklist is due (Saturday noon on) every gap is amber; from then on a real gap is red."""
@@ -901,9 +928,20 @@ class Engine:
saved = self.state.get("auto") or {}
self.auto_tries = {tuple(k.split("\0")): v for k, v in saved.get("tries", {}).items()}
self.auto_seen = {tuple(k.split("\0")): v for k, v in saved.get("seen", {}).items()}
# Failover: fixes marked "failover" run by themselves when their check fails (the stream's
# audio goes to FOH Backup when Studio B dies). One record per fix that fired, kept until the
# "undo" fix runs or the fault has been gone note_seconds; saved, so a restart never fires twice.
fo = cfg.get("failover") or {}
self.failover_cfg = fo
self.failover_windows = fo.get("windows", FAILOVER_WINDOWS)
self.failover_after = fo.get("after_seconds", 2)
self.failover_note = fo.get("note_seconds", 900)
self.failed = dict(self.state.get("failover") or {}) # fix id -> {"tile", "label", "note", "at", "faults", "cleared"}
self.failover_held = {} # fix id -> when the Dante-wide hold-back was last logged
self.notify = post_webhook # the alert message (replaced in tests)
self.autolog = AutoLog(self.auto.get("log_dir", "~/Desktop/Auto-Fix Logs"),
self.site.get("ingest_name", "NA Ingest"), windows_phrase(self.auto.get("windows", [])),
self.auto.get("keep_months", 1)) if self.auto else None
self.auto.get("keep_months", 1)) if (self.auto or fo) else None
if self.autolog:
self.autolog.cleanup(time.time())
self.auto_watch = {} # "BUTTON FAULT" auto-fix went after -> when (to log when it clears)
@@ -936,6 +974,7 @@ class Engine:
self.state.set("fix_last", {"\0".join(k): v for k, v in self.fix_last.items()})
self.state.set("auto", {"tries": {"\0".join(k): v for k, v in self.auto_tries.items()},
"seen": {"\0".join(k): v for k, v in self.auto_seen.items()}})
self.state.set("failover", self.failed)
def live_now(self, now):
"""(live?, why): changes to the panel wait while this is true. A recording HyperDeck is
@@ -1337,6 +1376,8 @@ class Engine:
waited = True
continue
self.fix_last[ident] = now
if kind == "fix":
self.failover_undone(what) # switching back by hand ends the failover note
self.remember()
if kind == "enable":
c = snap.conns.get(what)
@@ -1512,6 +1553,124 @@ class Engine:
self.remember()
return done
def failover_fix(self, key):
"""The failover fix a tracker key's check names (per sub-check, "*" for the rest), or None."""
tile_id, chk, sub = self.meta[key]
fix_id = chk.get("fix")
if isinstance(fix_id, dict):
fix_id = fix_id.get(sub, fix_id.get("*"))
fix = self.cfg["fixes"].get(fix_id) if fix_id else None
return fix_id if fix and fix.get("failover") else None
def dante_gone(self, conn):
"""How many devices on this Dante connection are gone right now, counted over every dante
check on every key (the same device may be watched by two keys)."""
return len({sub for k, (tid, chk, sub) in self.meta.items()
if chk["type"] == "dante" and chk["conn"] == conn and sub != "conn"
and self.trackers.get(k) and self.trackers[k].confirmed and self.trackers[k].result.level == ERR})
def failover_note_for(self, fix_id):
"""The note ("ON FOH BACKUP") of a failover that fired for this fix and is not switched back yet."""
rec = self.failed.get(fix_id)
return rec.get("note") if rec else None
def failover_undone(self, fix_id):
"""A fix that switches back by hand (the "undo" of a failover fix) was started: drop the
failover's note. If the fault is still there the failover fires again after_seconds later,
which is right: the stream must never stay silent."""
for fid in [f for f, r in self.failed.items() if self.cfg["fixes"].get(f, {}).get("undo") == fix_id]:
log.info("FAILOVER %s: switched back by hand (%s)", fid, fix_id)
del self.failed[fid]
def failover(self, snap, comp, now):
"""Run a "failover" fix by itself the moment its check fails for after_seconds (Studio B dies:
press FOH Backup, so the stream is not silent until someone notices). Once per fault episode,
only inside the failover windows, even while ALARMS is muted and even during a network blip
(a dead network is no reason to leave the stream silent). Never switches back by itself.
Timing, from a crash to the press: the Studio B Mac dies -> two missed 1-second pings (2-3 s)
+ 1 s debounce + 2 s after_seconds = about 5-6 s. Pro Tools crashes -> the 2-second ssh check
(2-3 s) + 1 s + 2 s = about 5-6 s. Its Dante device vanishing is seen as fast as Companion's
Dante module notices, then 3 s debounce + 2 s. Returns the fix ids it fired."""
if self.stopping:
return []
faults = {} # failover fix id -> [(tracker key, tracker)] in a confirmed fault
bad = set() # ...and those with a check that is bad at all (still debouncing)
for key in self.meta:
fix_id = self.failover_fix(key)
tr = self.trackers.get(key)
if not fix_id or not tr:
continue
if tr.confirmed and tr.result.level == ERR:
faults.setdefault(fix_id, []).append((key, tr))
# A check with no verdict (ssh could not ask, no ping result yet) is not a clear: a blip in
# the asking must not re-arm the failover and press a second time.
if tr.confirmed or tr.bad_since is not None or no_verdict(self.meta[key][1], snap, now):
bad.add(fix_id)
changed = False
for fix_id, rec in list(self.failed.items()):
if fix_id in bad:
# Still there, or back inside its debounce (the first seconds after a restart): not clear.
continue
if rec.get("cleared") is None: # the fault went: armed again, the note stays
rec["cleared"] = now
log.info("FAILOVER %s: fault clear - armed again (the stream stays %s until switched back by hand)",
fix_id, rec.get("note", "").lower())
changed = True
elif now - rec["cleared"] > self.failover_note:
del self.failed[fix_id]
changed = True
fired = []
for fix_id, items in faults.items():
rec = self.failed.get(fix_id)
if rec and (rec.get("cleared") is None or now - rec["cleared"] < REARM_SECONDS):
continue # fired for this fault already (or it only just cleared)
if not in_windows(self.failover_windows, now):
continue
lasting = [(k, tr) for k, tr in items if now - (tr.since or now) >= self.failover_after]
triggers = []
for k, tr in lasting:
chk = self.meta[k][1]
if chk["type"] == "dante" and self.dante_gone(chk["conn"]) > 1:
# Several Dante devices gone at once is NA Ingest's view of Dante, not that device.
if self.failover_held.get(fix_id, 0) < now - 600:
log.info("FAILOVER %s: %d Dante devices gone at once - that is NA Ingest's view of Dante, "
"not %s; not switching", fix_id, self.dante_gone(chk["conn"]), self.meta[k][2])
if self.autolog:
self.autolog.write(now, self.tiles[self.meta[k][0]]["label"],
"failover held back: %d Dante devices gone at once (NA Ingest's view of "
"Dante, not %s)" % (self.dante_gone(chk["conn"]), self.meta[k][2]))
self.failover_held[fix_id] = now
continue
triggers.append((k, tr))
if not triggers:
continue
key, tr = triggers[0]
tile_id, chk, sub = self.meta[key]
fix, label = self.cfg["fixes"][fix_id], self.tiles[tile_id]["label"]
msgs = list(dict.fromkeys(t.result.msg for _, t in items))
what = fix.get("label", fix_id)
self.failed[fix_id] = {"tile": tile_id, "label": label, "note": "ON " + fix.get("short", fix_id.upper()),
"at": now, "faults": msgs, "cleared": None}
self.fix_last[("fix", fix_id, "")] = now # a hold or Restart key right after says JUST TRIED / JUST DONE
log.info("FAILOVER %s: %s - %s", tile_id, ", ".join(msgs), what)
if self.autolog:
self.autolog.write(now, label, "FAILOVER: %s - %s" % (", ".join(msgs), what))
back = next((k["label"] for k in self.restart_keys.values() if k["fix"] == fix.get("undo")), None)
text = "%s: %s %s - switched over by itself: %s. Switch back by hand%s once it is fixed." % (
self.site.get("alert_prefix") or self.site.get("ingest_name") or "Status panel", label, ", ".join(msgs),
what, (" (hold %s on the Restart page)" % back) if back else "")
threading.Thread(target=self.notify, args=(self.cfg.get("alert_webhook"), text), daemon=True).start()
started = time.time()
self.run_plan(tile_id, [(("failover", fix_id), "fix", fix_id, chk)], snap, comp, now, cooldown=0, auto=True,
on_done=(lambda outcome, label=label, started=started: self.autolog.write(
now + (time.time() - started), label, "-> " + outcome)) if self.autolog else None)
fired.append(fix_id)
changed = True
if changed:
self.remember()
return fired
def describe(self, kind, what):
"""A plan item in words, for the auto-fix log."""
if kind == "restart":
@@ -1548,10 +1707,11 @@ class Engine:
self.note(key_id, "JUST DONE\n" + self.wait_text, "idle", now, 4)
return None
self.fix_last[ident] = now
self.failover_undone(fix_id)
self.remember()
fix = self.cfg["fixes"][fix_id]
log.info("%s %s: %s", verb, key_id, fix.get("label", fix_id))
self.note(key_id, "RESTARTING", "fixing", now, 180)
self.note(key_id, "PRESSING" if fix.get("kind") == "companion_press" else "RESTARTING", "fixing", now, 180)
def work():
r = self.executor(self.cfg, fix_id, chk or {})
@@ -1675,6 +1835,13 @@ def colored(text, level):
return {"x": text, "b": b, "f": f, "b2": b2, "f2": f2, "s": fit_size(text)}
def failover_line(engine, ids):
"""An extra last line ("ON FOH BACKUP") for a key one of whose tiles had a failover fire, until
it is switched back by hand (or the fault has been gone a while)."""
notes = list(dict.fromkeys(r["note"] for r in engine.failed.values() if r.get("tile") in ids and r.get("note")))
return ("\n" + ", ".join(notes)) if notes else ""
def render(engine, service_on):
out = {}
down_fixes, down_conns = engine.failing_targets()
@@ -1682,8 +1849,11 @@ def render(engine, service_on):
note = engine.notes.get(kid)
watch = [key["fix"]] + key.get("watch", [])
down = any(w[5:] in down_conns if w.startswith("conn:") else w in down_fixes for w in watch)
pressed = engine.failover_note_for(key["fix"])
if note and engine.now < note["until"]:
out[kid] = colored(key["label"] + "\n" + note["text"], note["level"])
elif pressed: # the failover pressed it already: say so, no flashing
out[kid] = colored(key["label"] + "\n" + pressed, "ok")
elif down: # flashes: this is the one to hold
out[kid] = colored(key["label"] + "\nHOLD TO FIX", "err")
else:
@@ -1736,7 +1906,7 @@ def render(engine, service_on):
text = tile["label"] + "\n" + first + ("\n+%d" % (len(faults) - 1) if len(faults) > 1 else "")
else:
text = tile["label"] + "\n" + ("OK" if level == "ok" else "OFF")
out[tid] = colored(text, level)
out[tid] = colored(text + failover_line(engine, tile["members"]), level)
continue
items = engine.items.get(tid, [])
level, faults = worst(items)
@@ -1748,7 +1918,7 @@ def render(engine, service_on):
text = tile["label"] + "\n" + (infos[0] if infos else "OFF")
else:
text = tile["label"] + "\n" + (infos[0] if infos else "OK")
out[tid] = colored(text, level)
out[tid] = colored(text + failover_line(engine, [tid]), level)
return out
@@ -1779,6 +1949,9 @@ def ping_once(host):
return False
_ping_lock = threading.Lock() # the 3-second and the 1-second ping pollers both update snap.ping
def poll_pings(snap, hosts, interface):
results = {}
@@ -1789,10 +1962,11 @@ def poll_pings(snap, hosts, interface):
t.start()
for t in threads:
t.join()
pings = dict(snap.ping)
for h, ok in results.items():
pings[h] = 0 if ok else pings.get(h, 0) + 1
snap.ping = pings
with _ping_lock:
pings = dict(snap.ping)
for h, ok in results.items():
pings[h] = 0 if ok else pings.get(h, 0) + 1
snap.ping = pings
if interface:
try:
out = subprocess.run(["/sbin/ifconfig", interface], capture_output=True, text=True, timeout=3).stdout
@@ -2149,24 +2323,61 @@ def parse_remote(out, n):
return [None if str(i) not in codes else codes[str(i)] == "0" for i in range(n)]
def poll_remote(snap, cfg):
by_host = {}
def remote_groups(cfg):
"""{(host, interval seconds): [commands]}: each machine's remote checks, grouped by how often they
ask to run ("interval_seconds"; Pro Tools on Studio B every 2 s, the rest every REMOTE_INTERVAL)."""
groups = {}
for t in all_tiles(cfg):
for c in t.get("checks", []):
if c["type"] == "remote":
by_host.setdefault(c["host"], [])
if c["cmd"] not in by_host[c["host"]]:
by_host[c["host"]].append(c["cmd"])
result = dict(snap.remote)
for host, cmds in by_host.items():
try:
out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds, host_shell(cfg, host))], capture_output=True,
text=True, timeout=20).stdout
except subprocess.TimeoutExpired:
out = ""
key = (c["host"], c.get("interval_seconds", REMOTE_INTERVAL))
groups.setdefault(key, [])
if c["cmd"] not in groups[key]:
groups[key].append(c["cmd"])
return groups
_remote_lock = threading.Lock()
_remote_last = {} # (host, interval) -> when that group last ran
_remote_running = {} # (host, interval) -> the thread running it now
def run_remote_group(snap, cfg, host, cmds):
"""One ssh for one group of commands; its verdicts go into snap.remote."""
try:
out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds, host_shell(cfg, host))], capture_output=True,
text=True, timeout=20).stdout
except (OSError, subprocess.SubprocessError):
out = ""
with _remote_lock:
result = dict(snap.remote)
for cmd, ok in zip(cmds, parse_remote(out, len(cmds))):
result["%s\0%s" % (host, cmd)] = ok # None = could not reach it: no verdict
snap.remote = result
snap.remote = result
def poll_remote(snap, cfg, now=None, last=None):
"""Run the remote check groups that are due, each in its own thread (one ssh process per group,
sharing the host's ControlMaster), so a dead machine (ssh gives up after 5 s) never delays the
2-second group on another. A group still running is not started again. With no `last` every
group runs and the call waits for them (run_once, tests). Returns the threads it started."""
now = time.time() if now is None else now
threads = []
for key, cmds in remote_groups(cfg).items():
if last is not None:
running = _remote_running.get(key)
if now - last.get(key, 0) < key[1] or (running and running.is_alive()):
continue
last[key] = now
t = threading.Thread(target=run_remote_group, args=(snap, cfg, key[0], cmds), name="remote-" + key[0], daemon=True)
t.start()
threads.append(t)
if last is not None:
_remote_running[key] = t
if last is None:
for t in threads:
t.join()
return threads
# --- The self-updating Dante map: name -> IP -> MAC -> switch port -> PoE --------
@@ -2325,6 +2536,18 @@ def execute_fix(cfg, fix_id, chk):
# Turn Companion's connection to the deck off and on: after a deck restarts, that is what
# brings it back green (a plain reconnect does not, 2026-10-11). The deck is never rebooted.
return reconnect_and_verify(cfg, fix, label, conn=chk["conn"], host=chk.get("ip"), toggle=True)
if kind == "companion_press":
# Press a button on one of Companion's own pages (the Audio page's FOH Backup), as a finger would.
status = make_companion(cfg).press(fix["page"], fix["row"], fix["col"])
where = "page %s row %s col %s" % (fix["page"], fix["row"], fix["col"])
if status == 200:
log.info("FIX %s -> pressed Companion %s", label, where)
return True
if status == 204:
log.info("FIX %s -> no button at %s (HTTP 204)", label, where)
return "NO BUTTON\nTHERE"
log.info("FIX %s -> HTTP %d", label, status)
return False
log.warning("FIX %s: unknown kind %r", label, kind)
except Exception as e:
log.warning("FIX %s failed: %s", label, e)
@@ -2653,8 +2876,18 @@ def load_config(path):
raise ValueError("fix %s: nothing to run" % fid)
if kind == "ready" and fix.get("action") not in READY_ACTIONS:
raise ValueError("fix %s: unknown READY action %s" % (fid, fix.get("action")))
if kind == "companion_press":
where = [fix.get("page"), fix.get("row"), fix.get("col")]
if (any(isinstance(v, bool) or not isinstance(v, int) for v in where)
or where[0] < 1 or where[1] < 0 or where[2] < 0):
raise ValueError("fix %s: a Companion button needs page, row and col (whole numbers)" % fid)
if fix.get("failover") and fix.get("never_auto"):
raise ValueError("fix %s: a failover fix runs by itself - it cannot be marked never to" % fid)
if fix.get("undo") and fix["undo"] not in cfg.get("fixes", {}):
raise ValueError("fix %s: unknown undo fix %s" % (fid, fix["undo"]))
check_windows(cfg.get("service_windows"), "service_windows")
check_windows((cfg.get("auto_fix") or {}).get("windows"), "auto_fix")
check_windows((cfg.get("failover") or {}).get("windows"), "failover")
check_windows((cfg.get("site") or {}).get("live_services"), "site.live_services")
ids = set()
for t in all_tiles(cfg):
@@ -2715,12 +2948,19 @@ def load_config(path):
return cfg
def ping_hosts(cfg, snap=None):
hosts = [ping_target(c, snap) if snap else c.get("host")
for t in all_tiles(cfg) for c in t.get("checks", []) if c["type"] == "ping"]
hosts = [h for h in hosts if h]
def ping_hosts(cfg, snap=None, fast=False):
"""The hosts to ping every PING_INTERVAL seconds (plus the gateway), or with fast=True the ones
whose check asks for less (Studio B every second, so the failover sees it die within 2-3 s)."""
quick, slow = [], []
for t in all_tiles(cfg):
for c in t.get("checks", []):
if c["type"] == "ping":
host = ping_target(c, snap) if snap else c.get("host")
(quick if c.get("interval_seconds", PING_INTERVAL) < PING_INTERVAL else slow).append(host)
if fast:
return list(dict.fromkeys(h for h in quick if h))
gw = cfg.get("network", {}).get("gateway")
return list(dict.fromkeys(hosts + ([gw] if gw else [])))
return list(dict.fromkeys(h for h in slow + ([gw] if gw else []) if h and h not in quick))
def http_checks(cfg):
@@ -2747,12 +2987,13 @@ def start_pollers(cfg, snap, stop, data_dir=None):
Periodic("slow-variables", 3.0, lambda: poll_variables(c3, snap, slow), stop),
Periodic("conn-hosts", 60.0, lambda: poll_conn_hosts(snap, cfg), stop),
Periodic("spotify", 3.0, lambda: poll_spotify(snap, cfg), stop),
Periodic("pings", 3.0, lambda: poll_pings(snap, ping_hosts(cfg, snap), iface), stop),
Periodic("pings", float(PING_INTERVAL), lambda: poll_pings(snap, ping_hosts(cfg, snap), iface), stop),
Periodic("fast-pings", 1.0, lambda: poll_pings(snap, ping_hosts(cfg, snap, fast=True), None), stop),
Periodic("midi", 0.5, lambda: poll_midi(snap, cfg), stop),
Periodic("system", 10.0, lambda: poll_system(snap, cfg), stop),
Periodic("bonjour", 15.0, lambda: poll_mdns(snap), stop),
Periodic("http", 5.0, lambda: poll_http(snap, http_checks(cfg)), stop),
Periodic("remote", 10.0, lambda: poll_remote(snap, cfg), stop),
Periodic("remote", 1.0, lambda: poll_remote(snap, cfg, last=_remote_last), stop), # runs the groups that are due
Periodic("dante-map", 30.0, lambda: poll_dante_map(snap, cfg, c4, dante_map_path(data_dir)), stop),
]
for t in threads:
@@ -2806,6 +3047,7 @@ def snapshot_summary(snap, engine, now):
"service_on": engine.service.on(now),
"alarms_override": engine.service.override,
"auto_fix": {"on": engine.auto_on, "paused": engine.auto_paused},
"failover": dict(engine.failed),
"burst": engine.burst and {"start": engine.burst["start"], "conns": sorted(engine.burst["conns"]),
"cleared": engine.burst["cleared"]},
"live": list(engine.live_now(now)),
@@ -2937,6 +3179,10 @@ def run(cfg, data_dir, dry_run=False, stop=None, config_sha=None):
engine.auto_fix(snap, comp, now)
except Exception: # a bug here must never take the panel down
log.exception("auto-fix failed; the panel carries on without it this round")
try:
engine.failover(snap, comp, now)
except Exception:
log.exception("failover failed; the panel carries on without it this round")
for kind, text in engine.events:
log.info("%s %s", kind, text)
engine.events = []
@@ -2997,8 +3243,9 @@ def run_once(cfg):
fast, slow = needed_variables(cfg)
poll_connections(comp, snap)
poll_variables(comp, snap, fast + slow)
poll_pings(snap, ping_hosts(cfg), cfg.get("network", {}).get("interface"))
poll_pings(snap, ping_hosts(cfg), None) # pings need two misses to count
everything = ping_hosts(cfg) + ping_hosts(cfg, fast=True)
poll_pings(snap, everything, cfg.get("network", {}).get("interface"))
poll_pings(snap, everything, None) # pings need two misses to count
poll_system(snap, cfg)
poll_mdns(snap)
poll_http(snap, http_checks(cfg))