397 lines
17 KiB
Python
397 lines
17 KiB
Python
|
"""Offline tests for the logic that decides what the team is told.
|
||
|
|
|
||
|
|
Run with: python -m unittest discover -s tests -v
|
||
|
|
|
||
|
|
No network, no API key. These cover version comparison, evidence
|
||
|
|
classification and the scoring rules, because those are the parts that can
|
||
|
|
break silently and still produce a confident-looking report.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import unittest
|
||
|
|
from datetime import timedelta
|
||
|
|
|
||
|
|
from avwatch.config import PolicyConfig
|
||
|
|
from avwatch.models import Evidence, Item, VersionCheck, utcnow
|
||
|
|
from avwatch.risk import heuristics
|
||
|
|
from avwatch.risk.classify import classify, relevance, severity
|
||
|
|
from avwatch.risk.collectors import clean_body, rank_and_filter
|
||
|
|
from avwatch.versions import bump_kind, compare, parse, pick_latest
|
||
|
|
|
||
|
|
|
||
|
|
class TestVersionParsing(unittest.TestCase):
|
||
|
|
def test_plain_versions(self):
|
||
|
|
self.assertEqual(parse("10.4.1").parts, (10, 4, 1))
|
||
|
|
self.assertEqual(parse("16.4").parts, (16, 4))
|
||
|
|
self.assertEqual(parse("v7.16.3").parts, (7, 16, 3))
|
||
|
|
self.assertEqual(parse("Version 2.2.5").parts, (2, 2, 5))
|
||
|
|
self.assertEqual(parse("firmware 9.0.2").parts, (9, 0, 2))
|
||
|
|
|
||
|
|
def test_invalid(self):
|
||
|
|
for bad in ("", None, "unknown", "latest", "n/a"):
|
||
|
|
self.assertFalse(parse(bad).valid, f"{bad!r} should not parse")
|
||
|
|
|
||
|
|
def test_ordering_across_widths(self):
|
||
|
|
# 10.4 vs 10.4.1 -- the classic off-by-one-segment trap
|
||
|
|
self.assertTrue(parse("10.4") < parse("10.4.1"))
|
||
|
|
self.assertTrue(parse("10.10") > parse("10.9"))
|
||
|
|
self.assertTrue(parse("2.0") > parse("1.99.99"))
|
||
|
|
self.assertEqual(parse("16.4"), parse("16.4.0"))
|
||
|
|
|
||
|
|
def test_prerelease_ranks_below_final(self):
|
||
|
|
self.assertTrue(parse("5.0.0-beta.2") < parse("5.0.0"))
|
||
|
|
self.assertTrue(parse("5.0.0-rc1") < parse("5.0.0"))
|
||
|
|
self.assertTrue(parse("5.0.0-alpha1") < parse("5.0.0-beta1"))
|
||
|
|
|
||
|
|
def test_compare(self):
|
||
|
|
self.assertEqual(compare("10.2", "10.4.1"), "behind")
|
||
|
|
self.assertEqual(compare("10.4.1", "10.4.1"), "current")
|
||
|
|
self.assertEqual(compare("10.5", "10.4.1"), "ahead")
|
||
|
|
self.assertEqual(compare(None, "10.4.1"), "unknown")
|
||
|
|
self.assertEqual(compare("garbage", "10.4.1"), "unknown")
|
||
|
|
|
||
|
|
def test_bump_kind(self):
|
||
|
|
self.assertEqual(bump_kind("31.0.1", "32.2.2"), "major")
|
||
|
|
self.assertEqual(bump_kind("10.2", "10.4.1"), "minor")
|
||
|
|
self.assertEqual(bump_kind("12.4.0", "12.4.1"), "patch")
|
||
|
|
self.assertEqual(bump_kind("12.4", "12.4"), "none")
|
||
|
|
|
||
|
|
def test_pick_latest_ignores_prereleases(self):
|
||
|
|
self.assertEqual(pick_latest(["7.15", "7.16.3", "7.14.1"]), "7.16.3")
|
||
|
|
self.assertEqual(pick_latest(["5.0.0", "5.1.0-beta.1"]), "5.0.0")
|
||
|
|
self.assertIsNone(pick_latest(["none", ""]))
|
||
|
|
|
||
|
|
|
||
|
|
class TestClassification(unittest.TestCase):
|
||
|
|
def test_brick_beats_crash(self):
|
||
|
|
sev, _ = severity("Device bricked after firmware update, crashes too")
|
||
|
|
self.assertEqual(sev, "brick")
|
||
|
|
|
||
|
|
def test_severity_tiers(self):
|
||
|
|
cases = [
|
||
|
|
("My ATEM won't boot after the update", "brick"),
|
||
|
|
("Console crashes randomly since 3.2", "crash"),
|
||
|
|
("No audio on channel 5 after update", "crash"),
|
||
|
|
("Had to roll back, this broke my presets", "regression"),
|
||
|
|
("Minor UI glitch in the new build", "annoyance"),
|
||
|
|
("New firmware released today", "info"),
|
||
|
|
]
|
||
|
|
for text, want in cases:
|
||
|
|
got, _ = severity(text)
|
||
|
|
self.assertEqual(got, want, f"{text!r} -> {got}, wanted {want}")
|
||
|
|
|
||
|
|
def test_howto_question_downgraded(self):
|
||
|
|
# Somebody asking for help is not a report that this release breaks.
|
||
|
|
for title in ("How do I fix a crash on startup?",
|
||
|
|
"Is it safe to update to 10.4.1?",
|
||
|
|
"Should I upgrade, anyone had problems?"):
|
||
|
|
_, sent, _ = classify(title)
|
||
|
|
self.assertNotEqual(sent, "negative", f"{title!r} should be softened")
|
||
|
|
|
||
|
|
def test_symptom_report_keeps_negative(self):
|
||
|
|
# A real failure report that happens to end in a question mark must
|
||
|
|
# NOT be softened -- that is the signal we are looking for.
|
||
|
|
for title in ("My ATEM won't boot after the update, what do I do?",
|
||
|
|
"Console crashes every time since 3.2 - anyone else?"):
|
||
|
|
_, sent, _ = classify(title)
|
||
|
|
self.assertEqual(sent, "negative", f"{title!r} should stay negative")
|
||
|
|
|
||
|
|
def test_sentiment(self):
|
||
|
|
_, sent, _ = classify("Avoid this update, it broke everything")
|
||
|
|
self.assertEqual(sent, "negative")
|
||
|
|
_, sent, _ = classify("Running 10.4.1, rock solid, no issues")
|
||
|
|
self.assertEqual(sent, "positive")
|
||
|
|
|
||
|
|
def test_relevance_rejects_offtopic(self):
|
||
|
|
terms = ["ATEM", "Constellation", "Blackmagic"]
|
||
|
|
on = relevance("ATEM Constellation firmware problem", terms, "10.4.1")
|
||
|
|
off = relevance("SKR1.4T 3D printer firmware upload", terms, "10.4.1")
|
||
|
|
self.assertGreater(on, 0.5)
|
||
|
|
self.assertLess(off, 0.34)
|
||
|
|
|
||
|
|
def test_version_mention_lifts_partial_relevance(self):
|
||
|
|
# relevance() saturates at 1.0 once every term matches, so the version
|
||
|
|
# boost is only observable when term coverage is partial. Ranking by
|
||
|
|
# version-specificity is handled separately by mentions_version --
|
||
|
|
# see TestEvidenceRanking.test_version_specific_ranks_first.
|
||
|
|
terms = ["Blackmagic", "Converters", "UpDownCross", "2110"]
|
||
|
|
with_v = relevance("Blackmagic 12.4 update", terms, "12.4")
|
||
|
|
without = relevance("Blackmagic update", terms, "12.4")
|
||
|
|
self.assertGreater(with_v, without)
|
||
|
|
self.assertLessEqual(with_v, 1.0)
|
||
|
|
|
||
|
|
|
||
|
|
class TestEvidenceRanking(unittest.TestCase):
|
||
|
|
def _ev(self, title, sev="crash", sent="negative", snippet=""):
|
||
|
|
return Evidence(source="github-issues", title=title, url=f"http://x/{title}",
|
||
|
|
snippet=snippet, severity=sev, sentiment=sent)
|
||
|
|
|
||
|
|
def test_offtopic_filtered_out(self):
|
||
|
|
evidence = [
|
||
|
|
self._ev("ATEM Constellation 10.4.1 crashes on boot"),
|
||
|
|
self._ev("Razer mouse firmware issue"),
|
||
|
|
]
|
||
|
|
kept = rank_and_filter(evidence, ["ATEM", "Constellation"], "10.4.1", limit=5)
|
||
|
|
self.assertEqual(len(kept), 1)
|
||
|
|
self.assertIn("ATEM", kept[0].title)
|
||
|
|
|
||
|
|
def test_version_specific_ranks_first(self):
|
||
|
|
evidence = [
|
||
|
|
self._ev("General crash report", sev="crash"),
|
||
|
|
self._ev("OBS Studio 32.2.2 profile import fails", sev="regression"),
|
||
|
|
]
|
||
|
|
kept = rank_and_filter(evidence, ["OBS", "Studio"], "32.2.2", limit=5)
|
||
|
|
self.assertTrue(kept[0].mentions_version)
|
||
|
|
|
||
|
|
def test_deduplicates_by_url(self):
|
||
|
|
a = self._ev("OBS Studio crash")
|
||
|
|
b = self._ev("OBS Studio crash")
|
||
|
|
kept = rank_and_filter([a, b], ["OBS", "Studio"], None, limit=5)
|
||
|
|
self.assertEqual(len(kept), 1)
|
||
|
|
|
||
|
|
def test_clean_body_strips_issue_template(self):
|
||
|
|
raw = ("### Operating System Info\nWindows 11\n\n### OBS Studio Version\n"
|
||
|
|
"32.2.2\n\n### Expected Behavior\n_No response_\n\n"
|
||
|
|
"```\nsome log\n```")
|
||
|
|
out = clean_body(raw)
|
||
|
|
self.assertNotIn("###", out)
|
||
|
|
self.assertNotIn("_No response_", out)
|
||
|
|
# The version must survive, or version matching breaks.
|
||
|
|
self.assertIn("32.2.2", out)
|
||
|
|
|
||
|
|
|
||
|
|
class TestHeuristics(unittest.TestCase):
|
||
|
|
def setUp(self):
|
||
|
|
self.policy = PolicyConfig()
|
||
|
|
|
||
|
|
def _item(self, **kw):
|
||
|
|
base = dict(id="test", name="Test Device", vendor="TestCo",
|
||
|
|
category="video", installed="1.0")
|
||
|
|
base.update(kw)
|
||
|
|
return Item(**base)
|
||
|
|
|
||
|
|
def _check(self, latest="2.0", age_days=None):
|
||
|
|
released = utcnow() - timedelta(days=age_days) if age_days is not None else None
|
||
|
|
return VersionCheck(item_id="test", latest=latest, released=released,
|
||
|
|
status="ok")
|
||
|
|
|
||
|
|
def _score(self, item, check, evidence=None, comparison="behind", bump=None):
|
||
|
|
if bump is None:
|
||
|
|
bump = bump_kind(item.installed, check.latest)
|
||
|
|
return heuristics.score(item, check, comparison, bump,
|
||
|
|
evidence or [], self.policy)
|
||
|
|
|
||
|
|
def test_freeze_always_avoid(self):
|
||
|
|
v = self._score(self._item(freeze=True), self._check())
|
||
|
|
self.assertEqual(v.recommendation, "avoid")
|
||
|
|
self.assertEqual(v.risk_score, 100.0)
|
||
|
|
|
||
|
|
def test_up_to_date_is_go(self):
|
||
|
|
item = self._item(installed="2.0")
|
||
|
|
v = self._score(item, self._check("2.0"), comparison="current")
|
||
|
|
self.assertEqual(v.recommendation, "go")
|
||
|
|
|
||
|
|
def test_missing_baseline_asks_for_it(self):
|
||
|
|
item = self._item(installed=None)
|
||
|
|
v = self._score(item, self._check("2.0"), comparison="unknown")
|
||
|
|
self.assertEqual(v.recommendation, "manual-check")
|
||
|
|
self.assertTrue(any("set-installed" in r for r in v.reasons))
|
||
|
|
|
||
|
|
def test_error_status_is_manual_check(self):
|
||
|
|
check = VersionCheck(item_id="test", status="error", detail="page changed")
|
||
|
|
v = self._score(self._item(), check, comparison="unknown")
|
||
|
|
self.assertEqual(v.recommendation, "manual-check")
|
||
|
|
|
||
|
|
def test_fresh_release_on_showstopper_waits(self):
|
||
|
|
item = self._item(criticality="showstopper", installed="1.9")
|
||
|
|
v = self._score(item, self._check("2.0", age_days=1))
|
||
|
|
self.assertIn(v.recommendation, ("wait", "avoid"))
|
||
|
|
self.assertIsNotNone(v.suggested_wait_days)
|
||
|
|
|
||
|
|
def test_aged_patch_on_low_criticality_is_go(self):
|
||
|
|
item = self._item(criticality="low", installed="1.0.1")
|
||
|
|
v = self._score(item, self._check("1.0.2", age_days=120))
|
||
|
|
self.assertEqual(v.recommendation, "go")
|
||
|
|
|
||
|
|
def test_brick_report_naming_version_forces_avoid(self):
|
||
|
|
item = self._item(criticality="high", installed="1.9")
|
||
|
|
ev = Evidence(source="reddit", title="2.0 bricked my unit", url="http://x",
|
||
|
|
severity="brick", sentiment="negative", mentions_version=True)
|
||
|
|
v = self._score(item, self._check("2.0", age_days=90), [ev])
|
||
|
|
self.assertEqual(v.recommendation, "avoid")
|
||
|
|
|
||
|
|
def test_generic_reports_discounted_vs_version_specific(self):
|
||
|
|
"""The calibration that matters: every active project has open crash
|
||
|
|
reports. Only ones naming the release are evidence about it."""
|
||
|
|
item = self._item(criticality="high", installed="1.9")
|
||
|
|
check = self._check("2.0", age_days=90)
|
||
|
|
|
||
|
|
generic = [Evidence(source="github-issues", title="crashes sometimes",
|
||
|
|
url=f"http://x/{i}", severity="crash",
|
||
|
|
sentiment="negative", mentions_version=False)
|
||
|
|
for i in range(4)]
|
||
|
|
specific = [Evidence(source="github-issues", title="2.0 crashes on load",
|
||
|
|
url=f"http://y/{i}", severity="crash",
|
||
|
|
sentiment="negative", mentions_version=True)
|
||
|
|
for i in range(4)]
|
||
|
|
|
||
|
|
v_generic = self._score(item, check, generic)
|
||
|
|
v_specific = self._score(item, check, specific)
|
||
|
|
self.assertLess(v_generic.risk_score, v_specific.risk_score)
|
||
|
|
self.assertTrue(any("background noise" in r for r in v_generic.reasons))
|
||
|
|
|
||
|
|
def test_criticality_raises_score(self):
|
||
|
|
check = self._check("2.0", age_days=5)
|
||
|
|
low = self._score(self._item(criticality="low", installed="1.9"), check)
|
||
|
|
crit = self._score(self._item(criticality="showstopper", installed="1.9"), check)
|
||
|
|
self.assertGreater(crit.risk_score, low.risk_score)
|
||
|
|
|
||
|
|
def test_dot_zero_release_penalised(self):
|
||
|
|
item = self._item(criticality="normal", installed="1.9")
|
||
|
|
v = self._score(item, self._check("2.0", age_days=60))
|
||
|
|
self.assertTrue(any(".0 release" in r for r in v.reasons))
|
||
|
|
|
||
|
|
def test_security_fix_lowers_risk(self):
|
||
|
|
item = self._item(criticality="normal", installed="1.9.1")
|
||
|
|
plain = self._check("1.9.2", age_days=5)
|
||
|
|
secure = self._check("1.9.2", age_days=5)
|
||
|
|
secure.release_notes = "Fixes a security vulnerability (CVE-2026-1234)."
|
||
|
|
self.assertLess(
|
||
|
|
self._score(item, secure).risk_score,
|
||
|
|
self._score(item, plain).risk_score,
|
||
|
|
)
|
||
|
|
|
||
|
|
def test_no_evidence_is_not_treated_as_safe(self):
|
||
|
|
item = self._item(criticality="normal", installed="1.9")
|
||
|
|
v = self._score(item, self._check("1.9.5", age_days=60), [])
|
||
|
|
self.assertTrue(any("absence of reports" in r for r in v.reasons))
|
||
|
|
|
||
|
|
def test_scores_stay_in_range(self):
|
||
|
|
for crit in ("low", "normal", "high", "showstopper"):
|
||
|
|
for age in (0, 1, 15, 400):
|
||
|
|
item = self._item(criticality=crit, installed="1.0")
|
||
|
|
ev = [Evidence(source="reddit", title="bricked", url="http://x",
|
||
|
|
severity="brick", sentiment="negative",
|
||
|
|
mentions_version=True)]
|
||
|
|
v = self._score(item, self._check("2.0", age_days=age), ev)
|
||
|
|
self.assertGreaterEqual(v.risk_score, 0.0)
|
||
|
|
self.assertLessEqual(v.risk_score, 100.0)
|
||
|
|
|
||
|
|
|
||
|
|
class TestInventoryEditing(unittest.TestCase):
|
||
|
|
def test_set_installed_preserves_comments(self):
|
||
|
|
import tempfile
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from avwatch import inventory
|
||
|
|
|
||
|
|
content = """\
|
||
|
|
# A header comment that must survive
|
||
|
|
defaults:
|
||
|
|
criticality: normal
|
||
|
|
|
||
|
|
items:
|
||
|
|
# a comment about this item
|
||
|
|
- id: thing-one
|
||
|
|
name: Thing One
|
||
|
|
vendor: Acme
|
||
|
|
category: video
|
||
|
|
installed:
|
||
|
|
source: {kind: manual}
|
||
|
|
|
||
|
|
- id: thing-two
|
||
|
|
name: Thing Two
|
||
|
|
vendor: Acme
|
||
|
|
category: audio
|
||
|
|
installed: '1.0'
|
||
|
|
source: {kind: manual}
|
||
|
|
"""
|
||
|
|
with tempfile.TemporaryDirectory() as tmp:
|
||
|
|
path = Path(tmp) / "inv.yaml"
|
||
|
|
path.write_text(content, encoding="utf-8")
|
||
|
|
|
||
|
|
inventory.save_installed(path, "thing-one", "3.1.4")
|
||
|
|
after = path.read_text(encoding="utf-8")
|
||
|
|
|
||
|
|
self.assertIn("# A header comment that must survive", after)
|
||
|
|
self.assertIn("# a comment about this item", after)
|
||
|
|
self.assertIn("installed: '3.1.4'", after)
|
||
|
|
# thing-two must be untouched
|
||
|
|
self.assertIn("installed: '1.0'", after)
|
||
|
|
|
||
|
|
items = {i.id: i for i in inventory.load(path)}
|
||
|
|
self.assertEqual(items["thing-one"].installed, "3.1.4")
|
||
|
|
self.assertEqual(items["thing-two"].installed, "1.0")
|
||
|
|
|
||
|
|
def test_unknown_id_raises(self):
|
||
|
|
import tempfile
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from avwatch import inventory
|
||
|
|
|
||
|
|
with tempfile.TemporaryDirectory() as tmp:
|
||
|
|
path = Path(tmp) / "inv.yaml"
|
||
|
|
path.write_text(
|
||
|
|
"items:\n - id: a\n name: A\n vendor: V\n"
|
||
|
|
" category: video\n source: {kind: manual}\n",
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
with self.assertRaises(inventory.InventoryError):
|
||
|
|
inventory.save_installed(path, "nope", "1.0")
|
||
|
|
|
||
|
|
|
||
|
|
class TestShippedInventory(unittest.TestCase):
|
||
|
|
"""Guards against a typo in inventory.yaml reaching a scheduled run."""
|
||
|
|
|
||
|
|
def test_loads_and_is_sane(self):
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from avwatch import inventory
|
||
|
|
|
||
|
|
path = Path(__file__).resolve().parent.parent / "inventory.yaml"
|
||
|
|
if not path.exists():
|
||
|
|
self.skipTest("inventory.yaml not present")
|
||
|
|
|
||
|
|
items = inventory.load(path)
|
||
|
|
self.assertGreater(len(items), 50)
|
||
|
|
|
||
|
|
ids = [i.id for i in items]
|
||
|
|
self.assertEqual(len(ids), len(set(ids)), "duplicate item ids")
|
||
|
|
|
||
|
|
for i in items:
|
||
|
|
if i.source_kind == "blackmagic":
|
||
|
|
self.assertIn("family", i.source, f"{i.id} missing family")
|
||
|
|
if i.source_kind == "github":
|
||
|
|
self.assertIn("repo", i.source, f"{i.id} missing repo")
|
||
|
|
self.assertIn("/", i.source["repo"], f"{i.id} repo not owner/name")
|
||
|
|
if i.source_kind == "html":
|
||
|
|
self.assertIn("url", i.source, f"{i.id} missing url")
|
||
|
|
self.assertIn("pattern", i.source, f"{i.id} missing pattern")
|
||
|
|
if i.source_kind == "websearch":
|
||
|
|
self.assertIn("query", i.source, f"{i.id} missing query")
|
||
|
|
|
||
|
|
def test_blackmagic_families_are_known(self):
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from avwatch import inventory
|
||
|
|
from avwatch.sources.blackmagic import FAMILY_PATTERNS
|
||
|
|
|
||
|
|
path = Path(__file__).resolve().parent.parent / "inventory.yaml"
|
||
|
|
if not path.exists():
|
||
|
|
self.skipTest("inventory.yaml not present")
|
||
|
|
|
||
|
|
for i in inventory.load(path):
|
||
|
|
if i.source_kind != "blackmagic":
|
||
|
|
continue
|
||
|
|
fam = i.source.get("family")
|
||
|
|
self.assertTrue(
|
||
|
|
fam in FAMILY_PATTERNS or "pattern" in i.source,
|
||
|
|
f"{i.id}: family {fam!r} is not a known BMD family and no "
|
||
|
|
f"explicit pattern was given",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main(verbosity=2)
|