Initial commit: AVWatch v1.0.0 with .gitignore and CLAUDE.md
This commit is contained in:
@@ -0,0 +1,396 @@
|
||||
"""Offline tests for the logic that decides what the team is told.
|
||||
|
||||
Run with: python -m unittest discover -s tests -v
|
||||
|
||||
No network, no API key. These cover version comparison, evidence
|
||||
classification and the scoring rules, because those are the parts that can
|
||||
break silently and still produce a confident-looking report.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import timedelta
|
||||
|
||||
from avwatch.config import PolicyConfig
|
||||
from avwatch.models import Evidence, Item, VersionCheck, utcnow
|
||||
from avwatch.risk import heuristics
|
||||
from avwatch.risk.classify import classify, relevance, severity
|
||||
from avwatch.risk.collectors import clean_body, rank_and_filter
|
||||
from avwatch.versions import bump_kind, compare, parse, pick_latest
|
||||
|
||||
|
||||
class TestVersionParsing(unittest.TestCase):
|
||||
def test_plain_versions(self):
|
||||
self.assertEqual(parse("10.4.1").parts, (10, 4, 1))
|
||||
self.assertEqual(parse("16.4").parts, (16, 4))
|
||||
self.assertEqual(parse("v7.16.3").parts, (7, 16, 3))
|
||||
self.assertEqual(parse("Version 2.2.5").parts, (2, 2, 5))
|
||||
self.assertEqual(parse("firmware 9.0.2").parts, (9, 0, 2))
|
||||
|
||||
def test_invalid(self):
|
||||
for bad in ("", None, "unknown", "latest", "n/a"):
|
||||
self.assertFalse(parse(bad).valid, f"{bad!r} should not parse")
|
||||
|
||||
def test_ordering_across_widths(self):
|
||||
# 10.4 vs 10.4.1 -- the classic off-by-one-segment trap
|
||||
self.assertTrue(parse("10.4") < parse("10.4.1"))
|
||||
self.assertTrue(parse("10.10") > parse("10.9"))
|
||||
self.assertTrue(parse("2.0") > parse("1.99.99"))
|
||||
self.assertEqual(parse("16.4"), parse("16.4.0"))
|
||||
|
||||
def test_prerelease_ranks_below_final(self):
|
||||
self.assertTrue(parse("5.0.0-beta.2") < parse("5.0.0"))
|
||||
self.assertTrue(parse("5.0.0-rc1") < parse("5.0.0"))
|
||||
self.assertTrue(parse("5.0.0-alpha1") < parse("5.0.0-beta1"))
|
||||
|
||||
def test_compare(self):
|
||||
self.assertEqual(compare("10.2", "10.4.1"), "behind")
|
||||
self.assertEqual(compare("10.4.1", "10.4.1"), "current")
|
||||
self.assertEqual(compare("10.5", "10.4.1"), "ahead")
|
||||
self.assertEqual(compare(None, "10.4.1"), "unknown")
|
||||
self.assertEqual(compare("garbage", "10.4.1"), "unknown")
|
||||
|
||||
def test_bump_kind(self):
|
||||
self.assertEqual(bump_kind("31.0.1", "32.2.2"), "major")
|
||||
self.assertEqual(bump_kind("10.2", "10.4.1"), "minor")
|
||||
self.assertEqual(bump_kind("12.4.0", "12.4.1"), "patch")
|
||||
self.assertEqual(bump_kind("12.4", "12.4"), "none")
|
||||
|
||||
def test_pick_latest_ignores_prereleases(self):
|
||||
self.assertEqual(pick_latest(["7.15", "7.16.3", "7.14.1"]), "7.16.3")
|
||||
self.assertEqual(pick_latest(["5.0.0", "5.1.0-beta.1"]), "5.0.0")
|
||||
self.assertIsNone(pick_latest(["none", ""]))
|
||||
|
||||
|
||||
class TestClassification(unittest.TestCase):
|
||||
def test_brick_beats_crash(self):
|
||||
sev, _ = severity("Device bricked after firmware update, crashes too")
|
||||
self.assertEqual(sev, "brick")
|
||||
|
||||
def test_severity_tiers(self):
|
||||
cases = [
|
||||
("My ATEM won't boot after the update", "brick"),
|
||||
("Console crashes randomly since 3.2", "crash"),
|
||||
("No audio on channel 5 after update", "crash"),
|
||||
("Had to roll back, this broke my presets", "regression"),
|
||||
("Minor UI glitch in the new build", "annoyance"),
|
||||
("New firmware released today", "info"),
|
||||
]
|
||||
for text, want in cases:
|
||||
got, _ = severity(text)
|
||||
self.assertEqual(got, want, f"{text!r} -> {got}, wanted {want}")
|
||||
|
||||
def test_howto_question_downgraded(self):
|
||||
# Somebody asking for help is not a report that this release breaks.
|
||||
for title in ("How do I fix a crash on startup?",
|
||||
"Is it safe to update to 10.4.1?",
|
||||
"Should I upgrade, anyone had problems?"):
|
||||
_, sent, _ = classify(title)
|
||||
self.assertNotEqual(sent, "negative", f"{title!r} should be softened")
|
||||
|
||||
def test_symptom_report_keeps_negative(self):
|
||||
# A real failure report that happens to end in a question mark must
|
||||
# NOT be softened -- that is the signal we are looking for.
|
||||
for title in ("My ATEM won't boot after the update, what do I do?",
|
||||
"Console crashes every time since 3.2 - anyone else?"):
|
||||
_, sent, _ = classify(title)
|
||||
self.assertEqual(sent, "negative", f"{title!r} should stay negative")
|
||||
|
||||
def test_sentiment(self):
|
||||
_, sent, _ = classify("Avoid this update, it broke everything")
|
||||
self.assertEqual(sent, "negative")
|
||||
_, sent, _ = classify("Running 10.4.1, rock solid, no issues")
|
||||
self.assertEqual(sent, "positive")
|
||||
|
||||
def test_relevance_rejects_offtopic(self):
|
||||
terms = ["ATEM", "Constellation", "Blackmagic"]
|
||||
on = relevance("ATEM Constellation firmware problem", terms, "10.4.1")
|
||||
off = relevance("SKR1.4T 3D printer firmware upload", terms, "10.4.1")
|
||||
self.assertGreater(on, 0.5)
|
||||
self.assertLess(off, 0.34)
|
||||
|
||||
def test_version_mention_lifts_partial_relevance(self):
|
||||
# relevance() saturates at 1.0 once every term matches, so the version
|
||||
# boost is only observable when term coverage is partial. Ranking by
|
||||
# version-specificity is handled separately by mentions_version --
|
||||
# see TestEvidenceRanking.test_version_specific_ranks_first.
|
||||
terms = ["Blackmagic", "Converters", "UpDownCross", "2110"]
|
||||
with_v = relevance("Blackmagic 12.4 update", terms, "12.4")
|
||||
without = relevance("Blackmagic update", terms, "12.4")
|
||||
self.assertGreater(with_v, without)
|
||||
self.assertLessEqual(with_v, 1.0)
|
||||
|
||||
|
||||
class TestEvidenceRanking(unittest.TestCase):
|
||||
def _ev(self, title, sev="crash", sent="negative", snippet=""):
|
||||
return Evidence(source="github-issues", title=title, url=f"http://x/{title}",
|
||||
snippet=snippet, severity=sev, sentiment=sent)
|
||||
|
||||
def test_offtopic_filtered_out(self):
|
||||
evidence = [
|
||||
self._ev("ATEM Constellation 10.4.1 crashes on boot"),
|
||||
self._ev("Razer mouse firmware issue"),
|
||||
]
|
||||
kept = rank_and_filter(evidence, ["ATEM", "Constellation"], "10.4.1", limit=5)
|
||||
self.assertEqual(len(kept), 1)
|
||||
self.assertIn("ATEM", kept[0].title)
|
||||
|
||||
def test_version_specific_ranks_first(self):
|
||||
evidence = [
|
||||
self._ev("General crash report", sev="crash"),
|
||||
self._ev("OBS Studio 32.2.2 profile import fails", sev="regression"),
|
||||
]
|
||||
kept = rank_and_filter(evidence, ["OBS", "Studio"], "32.2.2", limit=5)
|
||||
self.assertTrue(kept[0].mentions_version)
|
||||
|
||||
def test_deduplicates_by_url(self):
|
||||
a = self._ev("OBS Studio crash")
|
||||
b = self._ev("OBS Studio crash")
|
||||
kept = rank_and_filter([a, b], ["OBS", "Studio"], None, limit=5)
|
||||
self.assertEqual(len(kept), 1)
|
||||
|
||||
def test_clean_body_strips_issue_template(self):
|
||||
raw = ("### Operating System Info\nWindows 11\n\n### OBS Studio Version\n"
|
||||
"32.2.2\n\n### Expected Behavior\n_No response_\n\n"
|
||||
"```\nsome log\n```")
|
||||
out = clean_body(raw)
|
||||
self.assertNotIn("###", out)
|
||||
self.assertNotIn("_No response_", out)
|
||||
# The version must survive, or version matching breaks.
|
||||
self.assertIn("32.2.2", out)
|
||||
|
||||
|
||||
class TestHeuristics(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.policy = PolicyConfig()
|
||||
|
||||
def _item(self, **kw):
|
||||
base = dict(id="test", name="Test Device", vendor="TestCo",
|
||||
category="video", installed="1.0")
|
||||
base.update(kw)
|
||||
return Item(**base)
|
||||
|
||||
def _check(self, latest="2.0", age_days=None):
|
||||
released = utcnow() - timedelta(days=age_days) if age_days is not None else None
|
||||
return VersionCheck(item_id="test", latest=latest, released=released,
|
||||
status="ok")
|
||||
|
||||
def _score(self, item, check, evidence=None, comparison="behind", bump=None):
|
||||
if bump is None:
|
||||
bump = bump_kind(item.installed, check.latest)
|
||||
return heuristics.score(item, check, comparison, bump,
|
||||
evidence or [], self.policy)
|
||||
|
||||
def test_freeze_always_avoid(self):
|
||||
v = self._score(self._item(freeze=True), self._check())
|
||||
self.assertEqual(v.recommendation, "avoid")
|
||||
self.assertEqual(v.risk_score, 100.0)
|
||||
|
||||
def test_up_to_date_is_go(self):
|
||||
item = self._item(installed="2.0")
|
||||
v = self._score(item, self._check("2.0"), comparison="current")
|
||||
self.assertEqual(v.recommendation, "go")
|
||||
|
||||
def test_missing_baseline_asks_for_it(self):
|
||||
item = self._item(installed=None)
|
||||
v = self._score(item, self._check("2.0"), comparison="unknown")
|
||||
self.assertEqual(v.recommendation, "manual-check")
|
||||
self.assertTrue(any("set-installed" in r for r in v.reasons))
|
||||
|
||||
def test_error_status_is_manual_check(self):
|
||||
check = VersionCheck(item_id="test", status="error", detail="page changed")
|
||||
v = self._score(self._item(), check, comparison="unknown")
|
||||
self.assertEqual(v.recommendation, "manual-check")
|
||||
|
||||
def test_fresh_release_on_showstopper_waits(self):
|
||||
item = self._item(criticality="showstopper", installed="1.9")
|
||||
v = self._score(item, self._check("2.0", age_days=1))
|
||||
self.assertIn(v.recommendation, ("wait", "avoid"))
|
||||
self.assertIsNotNone(v.suggested_wait_days)
|
||||
|
||||
def test_aged_patch_on_low_criticality_is_go(self):
|
||||
item = self._item(criticality="low", installed="1.0.1")
|
||||
v = self._score(item, self._check("1.0.2", age_days=120))
|
||||
self.assertEqual(v.recommendation, "go")
|
||||
|
||||
def test_brick_report_naming_version_forces_avoid(self):
|
||||
item = self._item(criticality="high", installed="1.9")
|
||||
ev = Evidence(source="reddit", title="2.0 bricked my unit", url="http://x",
|
||||
severity="brick", sentiment="negative", mentions_version=True)
|
||||
v = self._score(item, self._check("2.0", age_days=90), [ev])
|
||||
self.assertEqual(v.recommendation, "avoid")
|
||||
|
||||
def test_generic_reports_discounted_vs_version_specific(self):
|
||||
"""The calibration that matters: every active project has open crash
|
||||
reports. Only ones naming the release are evidence about it."""
|
||||
item = self._item(criticality="high", installed="1.9")
|
||||
check = self._check("2.0", age_days=90)
|
||||
|
||||
generic = [Evidence(source="github-issues", title="crashes sometimes",
|
||||
url=f"http://x/{i}", severity="crash",
|
||||
sentiment="negative", mentions_version=False)
|
||||
for i in range(4)]
|
||||
specific = [Evidence(source="github-issues", title="2.0 crashes on load",
|
||||
url=f"http://y/{i}", severity="crash",
|
||||
sentiment="negative", mentions_version=True)
|
||||
for i in range(4)]
|
||||
|
||||
v_generic = self._score(item, check, generic)
|
||||
v_specific = self._score(item, check, specific)
|
||||
self.assertLess(v_generic.risk_score, v_specific.risk_score)
|
||||
self.assertTrue(any("background noise" in r for r in v_generic.reasons))
|
||||
|
||||
def test_criticality_raises_score(self):
|
||||
check = self._check("2.0", age_days=5)
|
||||
low = self._score(self._item(criticality="low", installed="1.9"), check)
|
||||
crit = self._score(self._item(criticality="showstopper", installed="1.9"), check)
|
||||
self.assertGreater(crit.risk_score, low.risk_score)
|
||||
|
||||
def test_dot_zero_release_penalised(self):
|
||||
item = self._item(criticality="normal", installed="1.9")
|
||||
v = self._score(item, self._check("2.0", age_days=60))
|
||||
self.assertTrue(any(".0 release" in r for r in v.reasons))
|
||||
|
||||
def test_security_fix_lowers_risk(self):
|
||||
item = self._item(criticality="normal", installed="1.9.1")
|
||||
plain = self._check("1.9.2", age_days=5)
|
||||
secure = self._check("1.9.2", age_days=5)
|
||||
secure.release_notes = "Fixes a security vulnerability (CVE-2026-1234)."
|
||||
self.assertLess(
|
||||
self._score(item, secure).risk_score,
|
||||
self._score(item, plain).risk_score,
|
||||
)
|
||||
|
||||
def test_no_evidence_is_not_treated_as_safe(self):
|
||||
item = self._item(criticality="normal", installed="1.9")
|
||||
v = self._score(item, self._check("1.9.5", age_days=60), [])
|
||||
self.assertTrue(any("absence of reports" in r for r in v.reasons))
|
||||
|
||||
def test_scores_stay_in_range(self):
|
||||
for crit in ("low", "normal", "high", "showstopper"):
|
||||
for age in (0, 1, 15, 400):
|
||||
item = self._item(criticality=crit, installed="1.0")
|
||||
ev = [Evidence(source="reddit", title="bricked", url="http://x",
|
||||
severity="brick", sentiment="negative",
|
||||
mentions_version=True)]
|
||||
v = self._score(item, self._check("2.0", age_days=age), ev)
|
||||
self.assertGreaterEqual(v.risk_score, 0.0)
|
||||
self.assertLessEqual(v.risk_score, 100.0)
|
||||
|
||||
|
||||
class TestInventoryEditing(unittest.TestCase):
|
||||
def test_set_installed_preserves_comments(self):
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from avwatch import inventory
|
||||
|
||||
content = """\
|
||||
# A header comment that must survive
|
||||
defaults:
|
||||
criticality: normal
|
||||
|
||||
items:
|
||||
# a comment about this item
|
||||
- id: thing-one
|
||||
name: Thing One
|
||||
vendor: Acme
|
||||
category: video
|
||||
installed:
|
||||
source: {kind: manual}
|
||||
|
||||
- id: thing-two
|
||||
name: Thing Two
|
||||
vendor: Acme
|
||||
category: audio
|
||||
installed: '1.0'
|
||||
source: {kind: manual}
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "inv.yaml"
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
inventory.save_installed(path, "thing-one", "3.1.4")
|
||||
after = path.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("# A header comment that must survive", after)
|
||||
self.assertIn("# a comment about this item", after)
|
||||
self.assertIn("installed: '3.1.4'", after)
|
||||
# thing-two must be untouched
|
||||
self.assertIn("installed: '1.0'", after)
|
||||
|
||||
items = {i.id: i for i in inventory.load(path)}
|
||||
self.assertEqual(items["thing-one"].installed, "3.1.4")
|
||||
self.assertEqual(items["thing-two"].installed, "1.0")
|
||||
|
||||
def test_unknown_id_raises(self):
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from avwatch import inventory
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "inv.yaml"
|
||||
path.write_text(
|
||||
"items:\n - id: a\n name: A\n vendor: V\n"
|
||||
" category: video\n source: {kind: manual}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self.assertRaises(inventory.InventoryError):
|
||||
inventory.save_installed(path, "nope", "1.0")
|
||||
|
||||
|
||||
class TestShippedInventory(unittest.TestCase):
|
||||
"""Guards against a typo in inventory.yaml reaching a scheduled run."""
|
||||
|
||||
def test_loads_and_is_sane(self):
|
||||
from pathlib import Path
|
||||
|
||||
from avwatch import inventory
|
||||
|
||||
path = Path(__file__).resolve().parent.parent / "inventory.yaml"
|
||||
if not path.exists():
|
||||
self.skipTest("inventory.yaml not present")
|
||||
|
||||
items = inventory.load(path)
|
||||
self.assertGreater(len(items), 50)
|
||||
|
||||
ids = [i.id for i in items]
|
||||
self.assertEqual(len(ids), len(set(ids)), "duplicate item ids")
|
||||
|
||||
for i in items:
|
||||
if i.source_kind == "blackmagic":
|
||||
self.assertIn("family", i.source, f"{i.id} missing family")
|
||||
if i.source_kind == "github":
|
||||
self.assertIn("repo", i.source, f"{i.id} missing repo")
|
||||
self.assertIn("/", i.source["repo"], f"{i.id} repo not owner/name")
|
||||
if i.source_kind == "html":
|
||||
self.assertIn("url", i.source, f"{i.id} missing url")
|
||||
self.assertIn("pattern", i.source, f"{i.id} missing pattern")
|
||||
if i.source_kind == "websearch":
|
||||
self.assertIn("query", i.source, f"{i.id} missing query")
|
||||
|
||||
def test_blackmagic_families_are_known(self):
|
||||
from pathlib import Path
|
||||
|
||||
from avwatch import inventory
|
||||
from avwatch.sources.blackmagic import FAMILY_PATTERNS
|
||||
|
||||
path = Path(__file__).resolve().parent.parent / "inventory.yaml"
|
||||
if not path.exists():
|
||||
self.skipTest("inventory.yaml not present")
|
||||
|
||||
for i in inventory.load(path):
|
||||
if i.source_kind != "blackmagic":
|
||||
continue
|
||||
fam = i.source.get("family")
|
||||
self.assertTrue(
|
||||
fam in FAMILY_PATTERNS or "pattern" in i.source,
|
||||
f"{i.id}: family {fam!r} is not a known BMD family and no "
|
||||
f"explicit pattern was given",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,364 @@
|
||||
"""Tests for version-coupling detection.
|
||||
|
||||
These cover the failure mode the tool exists to catch: not "the release was
|
||||
bad" but "half the matched set got flashed". All offline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import timedelta
|
||||
|
||||
from avwatch import coupling
|
||||
from avwatch.config import PolicyConfig
|
||||
from avwatch.models import Coupling, Finding, Item, VersionCheck, utcnow
|
||||
from avwatch.risk import heuristics
|
||||
from avwatch.versions import bump_kind, compare
|
||||
|
||||
|
||||
def item(iid, installed=None, **kw):
|
||||
base = dict(id=iid, name=iid.replace("-", " ").title(), vendor="TestCo",
|
||||
category="video", installed=installed)
|
||||
base.update(kw)
|
||||
return Item(**base)
|
||||
|
||||
|
||||
def finding(it, latest=None, age_days=90):
|
||||
released = utcnow() - timedelta(days=age_days) if latest else None
|
||||
chk = VersionCheck(item_id=it.id, latest=latest, released=released,
|
||||
status="ok" if latest else "manual")
|
||||
comp = compare(it.installed, latest) if latest else "unknown"
|
||||
bump = bump_kind(it.installed, latest) if latest else "unknown"
|
||||
f = Finding(item=it, check=chk, comparison=comp, bump=bump)
|
||||
f.verdict = heuristics.score(it, chk, comp, bump, [], PolicyConfig())
|
||||
return f
|
||||
|
||||
|
||||
def run(items_list, findings_list):
|
||||
by_id = {i.id: i for i in items_list}
|
||||
return coupling.annotate(findings_list, by_id), by_id
|
||||
|
||||
|
||||
class TestMustMatch(unittest.TestCase):
|
||||
def test_existing_mismatch_is_blocking(self):
|
||||
"""The switcher is already ahead of its panel -- that is broken now,
|
||||
not a future risk."""
|
||||
sw = item("atem", installed="10.4.1", criticality="showstopper",
|
||||
coupled_with=[Coupling("panel", "must_match",
|
||||
"panel drops offline")])
|
||||
panel = item("panel", installed="10.2", criticality="showstopper")
|
||||
fs, _ = run([sw, panel], [finding(sw, "10.4.1"), finding(panel, "10.4.1")])
|
||||
|
||||
sw_f = fs[0]
|
||||
blocking = [c for c in sw_f.coupling if c.severity == "blocking"]
|
||||
self.assertTrue(blocking, "a live mismatch must be blocking")
|
||||
self.assertIn("MISMATCH NOW", blocking[0].message)
|
||||
self.assertIn("10.2", blocking[0].message)
|
||||
|
||||
def test_matched_pair_with_update_pending_must_move_together(self):
|
||||
a = item("rx", installed="1.0", coupled_with=[
|
||||
Coupling("tx", "must_match", "they will not link")])
|
||||
b = item("tx", installed="1.0")
|
||||
fs, _ = run([a, b], [finding(a, "2.0"), finding(b, "2.0")])
|
||||
|
||||
blocking = [c for c in fs[0].coupling if c.severity == "blocking"]
|
||||
self.assertTrue(blocking)
|
||||
self.assertIn("same session", blocking[0].message)
|
||||
|
||||
def test_reverse_edge_is_mirrored(self):
|
||||
"""Only one half of a pair declares the edge; both halves must warn."""
|
||||
a = item("rx", installed="1.0", coupled_with=[
|
||||
Coupling("tx", "must_match")])
|
||||
b = item("tx", installed="2.0") # declares nothing
|
||||
fs, _ = run([a, b], [finding(a, "2.0"), finding(b, "2.0")])
|
||||
|
||||
b_f = next(f for f in fs if f.item.id == "tx")
|
||||
self.assertTrue(b_f.coupling,
|
||||
"the item that declares no edge must still be warned")
|
||||
|
||||
def test_auto_suppresses_the_do_them_together_reminder(self):
|
||||
"""ATEM Setup flashes switcher and panel in one pass, so reminding
|
||||
the team to do both is noise about an unskippable step."""
|
||||
sw = item("atem", installed="10.4", coupled_with=[
|
||||
Coupling("panel", "must_match", auto=True)])
|
||||
panel = item("panel", installed="10.4")
|
||||
fs, _ = run([sw, panel], [finding(sw, "10.5"), finding(panel, "10.5")])
|
||||
self.assertFalse(
|
||||
[c for c in fs[0].coupling if c.severity == "blocking"],
|
||||
"auto edges must not nag about doing both",
|
||||
)
|
||||
|
||||
def test_auto_still_reports_a_real_divergence(self):
|
||||
"""If an auto-flashed pair diverges, the automatic update FAILED --
|
||||
that must still be loud."""
|
||||
sw = item("atem", installed="10.4.1", coupled_with=[
|
||||
Coupling("panel", "must_match", auto=True)])
|
||||
panel = item("panel", installed="10.4")
|
||||
fs, _ = run([sw, panel],
|
||||
[finding(sw, "10.4.1"), finding(panel, "10.4.1")])
|
||||
blocking = [c for c in fs[0].coupling if c.severity == "blocking"]
|
||||
self.assertTrue(blocking, "a real divergence must still block")
|
||||
self.assertIn("MISMATCH NOW", blocking[0].message)
|
||||
|
||||
def test_non_auto_still_nags(self):
|
||||
sw = item("rx", installed="1.0", coupled_with=[
|
||||
Coupling("tx", "must_match")])
|
||||
tx = item("tx", installed="1.0")
|
||||
fs, _ = run([sw, tx], [finding(sw, "2.0"), finding(tx, "2.0")])
|
||||
self.assertTrue([c for c in fs[0].coupling if c.severity == "blocking"])
|
||||
|
||||
def test_different_numbering_schemes_flag_the_edge_not_the_gear(self):
|
||||
"""DiGiCo number the console (Stealth Core 22.0.2242) and its rack
|
||||
(1.11) on separate schemes. must_match there is a wrong edge, not a
|
||||
real fault, and must not cry BROKEN NOW every scan."""
|
||||
console = Item(id="console", name="Quantum 225", vendor="DiGiCo",
|
||||
category="audio", installed="22.0.2242",
|
||||
coupled_with=[Coupling("rack", "must_match")])
|
||||
rack = Item(id="rack", name="DQ Rack", vendor="DiGiCo",
|
||||
category="audio", installed="1.11")
|
||||
fs, _ = run([console, rack],
|
||||
[finding(console, "22.0.2242"), finding(rack, "1.11")])
|
||||
|
||||
sev = {c.severity for c in fs[0].coupling}
|
||||
self.assertNotIn("blocking", sev,
|
||||
"incomparable schemes must not report a real mismatch")
|
||||
self.assertIn("info", sev)
|
||||
self.assertTrue(any("numbering scheme" in c.message
|
||||
for c in fs[0].coupling))
|
||||
|
||||
def test_close_versions_still_report_a_real_mismatch(self):
|
||||
"""The guard must not swallow genuine mismatches."""
|
||||
a = item("atem", installed="10.4.1",
|
||||
coupled_with=[Coupling("panel", "must_match")])
|
||||
b = item("panel", installed="10.2")
|
||||
fs, _ = run([a, b], [finding(a, "10.4.1"), finding(b, "10.4.1")])
|
||||
self.assertTrue([c for c in fs[0].coupling if c.severity == "blocking"])
|
||||
|
||||
def test_unknown_installed_is_warning_not_blocking(self):
|
||||
a = item("rx", installed=None, coupled_with=[
|
||||
Coupling("tx", "must_match")])
|
||||
b = item("tx", installed="2.0")
|
||||
fs, _ = run([a, b], [finding(a, "2.0"), finding(b, "2.0")])
|
||||
sev = {c.severity for c in fs[0].coupling}
|
||||
self.assertIn("warning", sev)
|
||||
self.assertNotIn("blocking", sev)
|
||||
|
||||
def test_go_is_downgraded_when_coupled(self):
|
||||
"""The headline rule: a coupled item is never plain 'go'."""
|
||||
a = item("rx", installed="1.0", criticality="low", coupled_with=[
|
||||
Coupling("tx", "must_match")])
|
||||
b = item("tx", installed="1.0", criticality="low")
|
||||
fa = finding(a, "1.0.1", age_days=400) # aged patch on low crit
|
||||
self.assertEqual(fa.verdict.recommendation, "go")
|
||||
|
||||
fs, _ = run([a, b], [fa, finding(b, "1.0.1", age_days=400)])
|
||||
self.assertNotEqual(fs[0].verdict.recommendation, "go")
|
||||
self.assertTrue(any("Plan this as one job" in r
|
||||
for r in fs[0].verdict.reasons))
|
||||
|
||||
|
||||
class TestMustBeAtLeast(unittest.TestCase):
|
||||
def test_tool_behind_device_is_blocking(self):
|
||||
dev = item("qlxd4", installed="6.5", coupled_with=[
|
||||
Coupling("wwb", "must_be_at_least", "WWB pushes device firmware")])
|
||||
tool = item("wwb", installed="6.1")
|
||||
fs, _ = run([dev, tool], [finding(dev, "6.5"), finding(tool, "6.5")])
|
||||
|
||||
blocking = [c for c in fs[0].coupling if c.severity == "blocking"]
|
||||
self.assertTrue(blocking)
|
||||
self.assertIn("must be at or above", blocking[0].message)
|
||||
|
||||
def test_tool_ahead_is_fine(self):
|
||||
dev = item("qlxd4", installed="6.1", coupled_with=[
|
||||
Coupling("wwb", "must_be_at_least")])
|
||||
tool = item("wwb", installed="6.5")
|
||||
fs, _ = run([dev, tool], [finding(dev, "6.5"), finding(tool, "6.5")])
|
||||
self.assertFalse([c for c in fs[0].coupling if c.severity == "blocking"])
|
||||
|
||||
|
||||
class TestHostOf(unittest.TestCase):
|
||||
def test_major_platform_bump_blocks(self):
|
||||
os_item = item("macos", installed="14.0", coupled_with=[
|
||||
Coupling("protools", "host_of", "check Avid compatibility")])
|
||||
app = item("protools", installed="2024.3")
|
||||
fs, _ = run([os_item, app],
|
||||
[finding(os_item, "15.0"), finding(app, "2024.3")])
|
||||
blocking = [c for c in fs[0].coupling if c.severity == "blocking"]
|
||||
self.assertTrue(blocking)
|
||||
self.assertIn("platform change", blocking[0].message)
|
||||
|
||||
def test_patch_platform_bump_does_not_block(self):
|
||||
os_item = item("macos", installed="14.0", coupled_with=[
|
||||
Coupling("protools", "host_of")])
|
||||
app = item("protools", installed="2024.3")
|
||||
fs, _ = run([os_item, app],
|
||||
[finding(os_item, "14.0.1"), finding(app, "2024.3")])
|
||||
self.assertFalse([c for c in fs[0].coupling if c.severity == "blocking"])
|
||||
|
||||
def test_significant_any_blocks_on_a_patch(self):
|
||||
"""NETGEAR's M4250 puts breaking changes in the third digit, so
|
||||
13.0.4.x -> 13.0.5.x reads as a 'patch' but is anything but."""
|
||||
sw = item("switch", installed="13.0.4.26", criticality="showstopper",
|
||||
coupled_with=[Coupling("dante", "host_of",
|
||||
note="the whole Dante network",
|
||||
significant="any")])
|
||||
dante = item("dante", installed="4.8")
|
||||
fs, _ = run([sw, dante],
|
||||
[finding(sw, "13.0.5.26"), finding(dante, "4.8")])
|
||||
|
||||
self.assertEqual(fs[0].bump, "patch", "sanity: reads as a patch")
|
||||
blocking = [c for c in fs[0].coupling if c.severity == "blocking"]
|
||||
self.assertTrue(blocking, "significant: any must still block")
|
||||
self.assertIn("Any update", blocking[0].message)
|
||||
|
||||
def test_significant_major_stays_quiet_on_a_patch(self):
|
||||
sw = item("switch", installed="13.0.4.26", coupled_with=[
|
||||
Coupling("dante", "host_of")]) # default: major
|
||||
dante = item("dante", installed="4.8")
|
||||
fs, _ = run([sw, dante],
|
||||
[finding(sw, "13.0.5.26"), finding(dante, "4.8")])
|
||||
self.assertFalse([c for c in fs[0].coupling if c.severity == "blocking"])
|
||||
|
||||
|
||||
class TestCarrierAndContent(unittest.TestCase):
|
||||
def test_carrier_warns_when_device_behind(self):
|
||||
tool = item("suu", installed="1.0", coupled_with=[
|
||||
Coupling("ad4q", "carries_firmware_for")])
|
||||
dev = item("ad4q", installed="1.0")
|
||||
fs, _ = run([tool, dev], [finding(tool, "1.0"), finding(dev, "2.0")])
|
||||
msgs = [c.message for c in fs[0].coupling]
|
||||
self.assertTrue(any("Update this tool first" in m for m in msgs))
|
||||
|
||||
def test_content_warns_when_engine_behind(self):
|
||||
show = item("vista-show", installed="1.0", coupled_with=[
|
||||
Coupling("vista", "content_for")])
|
||||
engine = item("vista", installed="3.1")
|
||||
fs, _ = run([show, engine], [finding(show, "1.0"), finding(engine, "3.2")])
|
||||
msgs = [c.message for c in fs[0].coupling]
|
||||
self.assertTrue(any("Back this up first" in m for m in msgs))
|
||||
|
||||
|
||||
class TestValidation(unittest.TestCase):
|
||||
def test_dangling_id_reported(self):
|
||||
a = item("a", coupled_with=[Coupling("nope", "must_match")])
|
||||
problems = coupling.validate({"a": a})
|
||||
self.assertTrue(any("not an inventory id" in p for p in problems))
|
||||
|
||||
def test_unknown_relation_reported(self):
|
||||
a = item("a", coupled_with=[Coupling("b", "sometimes_maybe")])
|
||||
b = item("b")
|
||||
problems = coupling.validate({"a": a, "b": b})
|
||||
self.assertTrue(any("unknown relation" in p for p in problems))
|
||||
|
||||
def test_self_coupling_reported(self):
|
||||
a = item("a", coupled_with=[Coupling("a", "must_match")])
|
||||
problems = coupling.validate({"a": a})
|
||||
self.assertTrue(any("coupled to itself" in p for p in problems))
|
||||
|
||||
def test_dangling_id_is_info_not_blocking(self):
|
||||
a = item("a", installed="1.0", coupled_with=[
|
||||
Coupling("nope", "must_match")])
|
||||
fs, _ = run([a], [finding(a, "2.0")])
|
||||
self.assertEqual([c.severity for c in fs[0].coupling], ["info"])
|
||||
|
||||
def test_groups_only_returns_real_sets(self):
|
||||
a = item("a", update_group="atem")
|
||||
b = item("b", update_group="atem")
|
||||
c = item("c", update_group="lonely")
|
||||
grps = coupling.groups({"a": a, "b": b, "c": c})
|
||||
self.assertIn("atem", grps)
|
||||
self.assertNotIn("lonely", grps, "a group of one is not a set")
|
||||
|
||||
|
||||
class TestUpdateClasses(unittest.TestCase):
|
||||
"""Rows where 'should we install it?' is the wrong question."""
|
||||
|
||||
def setUp(self):
|
||||
self.policy = PolicyConfig()
|
||||
|
||||
def _v(self, it, latest="2.0"):
|
||||
chk = VersionCheck(item_id=it.id, latest=latest, status="ok")
|
||||
return heuristics.score(it, chk, "behind", "major", [], self.policy)
|
||||
|
||||
def test_do_not_touch_never_recommended(self):
|
||||
it = item("fire-alarm", installed="1.0", criticality="showstopper",
|
||||
update_class="do_not_touch")
|
||||
v = self._v(it)
|
||||
self.assertEqual(v.recommendation, "avoid")
|
||||
self.assertTrue(any("contractor" in r for r in v.reasons))
|
||||
|
||||
def test_vendor_pushed_gives_no_go_no_go(self):
|
||||
it = item("youtube", installed="1.0", update_class="vendor_pushed")
|
||||
v = self._v(it)
|
||||
self.assertEqual(v.recommendation, "manual-check")
|
||||
self.assertTrue(any("whether or not" in r or "no install decision" in r
|
||||
for r in v.reasons))
|
||||
|
||||
def test_expiry_class_mentions_date(self):
|
||||
it = item("ccli", installed="1.0", update_class="expiry",
|
||||
expires="2027-03-01")
|
||||
v = self._v(it)
|
||||
self.assertTrue(any("2027-03-01" in r for r in v.reasons))
|
||||
|
||||
def test_unconfirmed_gives_no_advice(self):
|
||||
it = item("main-pa", installed=None, criticality="high",
|
||||
status="unconfirmed")
|
||||
v = self._v(it)
|
||||
self.assertEqual(v.recommendation, "manual-check")
|
||||
self.assertTrue(any("guessed product" in r for r in v.reasons))
|
||||
|
||||
|
||||
class TestShippedCouplingGraph(unittest.TestCase):
|
||||
"""The real inventory files must have a sound coupling graph."""
|
||||
|
||||
def _items(self):
|
||||
from pathlib import Path
|
||||
|
||||
from avwatch import inventory
|
||||
|
||||
path = Path(__file__).resolve().parent.parent / "inventory.yaml"
|
||||
if not path.exists():
|
||||
self.skipTest("inventory.yaml not present")
|
||||
return inventory.by_id(inventory.load(path))
|
||||
|
||||
def test_graph_resolves(self):
|
||||
problems = coupling.validate(self._items())
|
||||
self.assertEqual(problems, [], "coupling graph has dangling edges")
|
||||
|
||||
def test_must_match_edges_are_reciprocal_or_mirrored(self):
|
||||
"""Not a hard requirement (annotate mirrors them), but every
|
||||
must_match target should at least exist and share the update_group."""
|
||||
items = self._items()
|
||||
for it in items.values():
|
||||
for link in it.coupled_with:
|
||||
if link.relation != "must_match":
|
||||
continue
|
||||
other = items[link.id]
|
||||
if it.update_group and other.update_group:
|
||||
self.assertEqual(
|
||||
it.update_group, other.update_group,
|
||||
f"{it.id} and {other.id} must_match but are in "
|
||||
f"different update groups",
|
||||
)
|
||||
|
||||
def test_unconfirmed_items_have_no_query(self):
|
||||
"""An unconfirmed row must not carry a search query -- that is how a
|
||||
guessed vendor's version reaches the report."""
|
||||
from pathlib import Path
|
||||
|
||||
from avwatch import inventory
|
||||
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
for f in inventory.resolve_paths(root / "inventory.yaml"):
|
||||
for it in inventory.load(f):
|
||||
if it.is_confirmed:
|
||||
continue
|
||||
q = (it.source or {}).get("query", "")
|
||||
self.assertFalse(
|
||||
str(q).strip(),
|
||||
f"{it.id} in {f.name} is unconfirmed but has query {q!r}",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user