190 lines
6.8 KiB
Python
190 lines
6.8 KiB
Python
|
"""Scan orchestration: for every item, find the latest version, gather
|
||
|
|
evidence, and produce a verdict."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import uuid
|
||
|
|
from datetime import timezone
|
||
|
|
|
||
|
|
from . import coupling
|
||
|
|
from .config import Config
|
||
|
|
from .http import HttpClient
|
||
|
|
from .models import Finding, Item, RunSummary, utcnow
|
||
|
|
from .risk import collectors, decide, gather_evidence
|
||
|
|
from .risk.llm import ClaudeVerdictEngine
|
||
|
|
from .sources import check_item
|
||
|
|
from .sources.base import Context
|
||
|
|
from .store import Store
|
||
|
|
from .versions import bump_kind, compare
|
||
|
|
|
||
|
|
log = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
class Scanner:
|
||
|
|
def __init__(self, cfg: Config, store: Store, offline: bool = False):
|
||
|
|
self.cfg = cfg
|
||
|
|
self.store = store
|
||
|
|
self.http = HttpClient(
|
||
|
|
cache_dir=cfg.cache_dir,
|
||
|
|
cache_ttl=cfg.cache_ttl,
|
||
|
|
timeout=cfg.http_timeout,
|
||
|
|
offline=offline,
|
||
|
|
)
|
||
|
|
# The websearch source reads its config and the lookup cache out of
|
||
|
|
# the shared scratch, so it can stay a plain Source like the others.
|
||
|
|
self.ctx = Context(http=self.http, scratch={
|
||
|
|
"lookup_cfg": cfg.lookup,
|
||
|
|
"store": store,
|
||
|
|
"lookup_calls": 0,
|
||
|
|
})
|
||
|
|
self.engine = ClaudeVerdictEngine(cfg.llm)
|
||
|
|
self.reddit = collectors.RedditClient(
|
||
|
|
self.http, cfg.risk.reddit_client_id, cfg.risk.reddit_client_secret
|
||
|
|
)
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------
|
||
|
|
|
||
|
|
def scan(
|
||
|
|
self,
|
||
|
|
items: list[Item],
|
||
|
|
mode: str = "scan",
|
||
|
|
llm_policy: str = "new-only",
|
||
|
|
only: list[str] | None = None,
|
||
|
|
) -> tuple[list[Finding], RunSummary]:
|
||
|
|
"""Run a full pass.
|
||
|
|
|
||
|
|
``llm_policy``:
|
||
|
|
new-only -- ask Claude only about versions seen for the first time
|
||
|
|
(the cheap default for mid-week scans)
|
||
|
|
all -- ask about every item we are behind on (the Monday digest)
|
||
|
|
none -- heuristics only
|
||
|
|
"""
|
||
|
|
if only:
|
||
|
|
wanted = set(only)
|
||
|
|
items = [i for i in items if i.id in wanted]
|
||
|
|
|
||
|
|
if llm_policy == "none":
|
||
|
|
# --no-llm means no paid calls at all, lookups included.
|
||
|
|
self.ctx.scratch["lookup_cfg"] = None
|
||
|
|
if self.http.offline:
|
||
|
|
self.ctx.scratch["lookup_cfg"] = None
|
||
|
|
|
||
|
|
run = RunSummary(
|
||
|
|
run_id=uuid.uuid4().hex[:12], started=utcnow(), mode=mode
|
||
|
|
)
|
||
|
|
self.store.start_run(run)
|
||
|
|
|
||
|
|
findings: list[Finding] = []
|
||
|
|
for n, item in enumerate(items, 1):
|
||
|
|
log.info("[%d/%d] %s", n, len(items), item.name)
|
||
|
|
findings.append(self._scan_one(item, run.run_id, llm_policy))
|
||
|
|
|
||
|
|
# Coupling has to run after every item is scanned -- it compares
|
||
|
|
# items against each other, so it cannot be done item-by-item.
|
||
|
|
by_id = {i.id: i for i in items}
|
||
|
|
problems = coupling.validate(by_id)
|
||
|
|
for p in problems:
|
||
|
|
log.warning("coupling: %s", p)
|
||
|
|
coupling.annotate(findings, by_id)
|
||
|
|
n_coupled = sum(1 for f in findings if f.coupling)
|
||
|
|
if n_coupled:
|
||
|
|
log.info("coupling: %d item(s) carry version dependencies, "
|
||
|
|
"%d blocking", n_coupled,
|
||
|
|
sum(1 for f in findings if f.has_blocking_coupling))
|
||
|
|
# Re-save so the stored verdict reflects the downgrade.
|
||
|
|
for f in findings:
|
||
|
|
if f.coupling:
|
||
|
|
self.store.save_finding(run.run_id, f)
|
||
|
|
|
||
|
|
run.finished = utcnow()
|
||
|
|
run.items_checked = len(findings)
|
||
|
|
run.updates_found = sum(1 for f in findings if f.comparison == "behind")
|
||
|
|
run.errors = sum(1 for f in findings if f.check.status == "error")
|
||
|
|
run.new_since_last = sum(1 for f in findings if f.is_new_since_last_run)
|
||
|
|
run.http_requests = self.http.stats["requests"]
|
||
|
|
run.cache_hits = self.http.stats["cache_hits"]
|
||
|
|
self.store.finish_run(run)
|
||
|
|
|
||
|
|
log.info(
|
||
|
|
"run %s: %d checked, %d behind, %d new, %d errors "
|
||
|
|
"(%d requests, %d cache hits, %d verdict calls, %d web lookups)",
|
||
|
|
run.run_id, run.items_checked, run.updates_found, run.new_since_last,
|
||
|
|
run.errors, run.http_requests, run.cache_hits, self.engine.calls,
|
||
|
|
self.ctx.scratch.get("lookup_calls", 0),
|
||
|
|
)
|
||
|
|
return findings, run
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------
|
||
|
|
|
||
|
|
def _scan_one(self, item: Item, run_id: str, llm_policy: str) -> Finding:
|
||
|
|
check = check_item(item, self.ctx)
|
||
|
|
is_new = self.store.record_sighting(check)
|
||
|
|
|
||
|
|
comparison = (
|
||
|
|
compare(item.installed, check.latest)
|
||
|
|
if check.status == "ok" else "unknown"
|
||
|
|
)
|
||
|
|
bump = (
|
||
|
|
bump_kind(item.installed, check.latest)
|
||
|
|
if check.status == "ok" else "unknown"
|
||
|
|
)
|
||
|
|
|
||
|
|
finding = Finding(
|
||
|
|
item=item, check=check, comparison=comparison, bump=bump,
|
||
|
|
is_new_since_last_run=is_new,
|
||
|
|
)
|
||
|
|
|
||
|
|
# Community scan: only where it can change the answer.
|
||
|
|
want_evidence = self.cfg.risk.enabled and (
|
||
|
|
comparison == "behind"
|
||
|
|
or (check.status == "manual" and not self.cfg.risk.only_when_behind)
|
||
|
|
or not self.cfg.risk.only_when_behind
|
||
|
|
)
|
||
|
|
if want_evidence:
|
||
|
|
finding.evidence = gather_evidence(
|
||
|
|
item, check, self.cfg, self.http, self.reddit
|
||
|
|
)
|
||
|
|
|
||
|
|
# How long has this version been known to exist? Used when the vendor
|
||
|
|
# publishes no release date.
|
||
|
|
first_seen_age = None
|
||
|
|
if check.latest:
|
||
|
|
seen = self.store.first_seen(item.id, check.latest)
|
||
|
|
if seen:
|
||
|
|
first_seen_age = (
|
||
|
|
utcnow() - seen.astimezone(timezone.utc)
|
||
|
|
).total_seconds() / 86400.0
|
||
|
|
|
||
|
|
use_llm = (
|
||
|
|
llm_policy == "all"
|
||
|
|
or (llm_policy == "new-only" and is_new)
|
||
|
|
)
|
||
|
|
finding.verdict = decide(
|
||
|
|
item, check, comparison, bump, finding.evidence, self.cfg,
|
||
|
|
self.engine, first_seen_age, use_llm=use_llm,
|
||
|
|
)
|
||
|
|
|
||
|
|
self.store.save_finding(run_id, finding)
|
||
|
|
return finding
|
||
|
|
|
||
|
|
|
||
|
|
def sort_findings(findings: list[Finding]) -> list[Finding]:
|
||
|
|
"""Most-actionable first: things that need a decision, worst risk on top,
|
||
|
|
then show-critical gear, then everything else."""
|
||
|
|
rec_rank = {"avoid": 0, "wait": 1, "manual-check": 2, "go": 3, "unknown": 4}
|
||
|
|
crit_rank = {"showstopper": 0, "high": 1, "normal": 2, "low": 3}
|
||
|
|
|
||
|
|
def key(f: Finding):
|
||
|
|
v = f.verdict
|
||
|
|
behind = 0 if f.comparison == "behind" else 1
|
||
|
|
return (
|
||
|
|
behind,
|
||
|
|
rec_rank.get(v.recommendation if v else "unknown", 4),
|
||
|
|
crit_rank.get(f.item.criticality, 2),
|
||
|
|
-(v.risk_score if v else 0.0),
|
||
|
|
f.item.name.lower(),
|
||
|
|
)
|
||
|
|
|
||
|
|
return sorted(findings, key=key)
|