Initial commit: AVWatch v1.0.0 with .gitignore and CLAUDE.md
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
"""Source registry and dispatch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from ..models import Item, VersionCheck
|
||||
from .base import Context, error
|
||||
from .blackmagic import BlackmagicSource
|
||||
from .generic import HtmlSource, JsonSource, ManualSource, RssSource
|
||||
from .github_releases import GithubSource
|
||||
from .websearch import WebSearchSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_SOURCES = {
|
||||
s.kind: s()
|
||||
for s in (
|
||||
BlackmagicSource,
|
||||
GithubSource,
|
||||
HtmlSource,
|
||||
JsonSource,
|
||||
RssSource,
|
||||
WebSearchSource,
|
||||
ManualSource,
|
||||
)
|
||||
}
|
||||
|
||||
KINDS = sorted(_SOURCES)
|
||||
|
||||
|
||||
def check_item(item: Item, ctx: Context) -> VersionCheck:
|
||||
"""Resolve the latest available version for one item. Never raises --
|
||||
an adapter blowing up must not abort a 90-item scan."""
|
||||
# An unconfirmed placeholder must not be searched on. A query built from
|
||||
# a guessed vendor returns a real, correct-looking version for the WRONG
|
||||
# product, and the team then acts on it. Silence is the safe answer until
|
||||
# somebody names the actual model.
|
||||
if not item.is_confirmed:
|
||||
return VersionCheck(
|
||||
item_id=item.id,
|
||||
status="unconfigured",
|
||||
detail=("Make/model not confirmed, so no version lookup is "
|
||||
"attempted -- a guessed vendor would return the wrong "
|
||||
"product's version. Set status: confirmed once you have "
|
||||
"filled in the real make and model."),
|
||||
)
|
||||
|
||||
kind = item.source_kind
|
||||
|
||||
# Nothing to decide, so nothing worth looking up.
|
||||
if item.update_class == "do_not_touch":
|
||||
return VersionCheck(
|
||||
item_id=item.id,
|
||||
status="unconfigured",
|
||||
detail=("Marked do_not_touch, so no version is looked up -- this "
|
||||
"is a licensed contractor's to update, never ours. "
|
||||
"Tracked for visibility only."),
|
||||
)
|
||||
|
||||
# Vendor-pushed items update on someone else's schedule. A paid lookup
|
||||
# buys a version number nobody can act on, so skip it -- but keep free
|
||||
# sources (GitHub, the Blackmagic feed), which cost nothing and do at
|
||||
# least tell us a change landed.
|
||||
if item.update_class == "vendor_pushed" and kind == "websearch":
|
||||
return VersionCheck(
|
||||
item_id=item.id,
|
||||
status="unconfigured",
|
||||
detail=("Marked vendor_pushed, so the paid version lookup is "
|
||||
"skipped -- it updates on the supplier's schedule and "
|
||||
"there is no install decision to inform. Watch for "
|
||||
"breaking-change notices instead."),
|
||||
)
|
||||
source = _SOURCES.get(kind)
|
||||
if source is None:
|
||||
return error(item, f"unknown source kind {kind!r}; known: {', '.join(KINDS)}")
|
||||
try:
|
||||
return source.check(item, ctx)
|
||||
except Exception as exc: # noqa: BLE001 - deliberate isolation boundary
|
||||
log.exception("source %s failed for %s", kind, item.id)
|
||||
return error(item, f"{type(exc).__name__}: {exc}")
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Source protocol shared by every version adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
from ..http import HttpClient
|
||||
from ..models import Item, VersionCheck
|
||||
|
||||
|
||||
@dataclass
|
||||
class Context:
|
||||
"""Passed to every source. ``scratch`` lets an adapter fetch one shared
|
||||
payload (a vendor-wide downloads feed, say) and reuse it across the dozens
|
||||
of items that resolve from it."""
|
||||
|
||||
http: HttpClient
|
||||
scratch: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def memo(self, key: str, factory):
|
||||
if key not in self.scratch:
|
||||
self.scratch[key] = factory()
|
||||
return self.scratch[key]
|
||||
|
||||
|
||||
class Source(Protocol):
|
||||
kind: str
|
||||
|
||||
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||||
...
|
||||
|
||||
|
||||
def error(item: Item, detail: str) -> VersionCheck:
|
||||
return VersionCheck(item_id=item.id, status="error", detail=detail)
|
||||
|
||||
|
||||
def manual(item: Item, detail: str = "No machine-readable feed; check by hand.") -> VersionCheck:
|
||||
return VersionCheck(item_id=item.id, status="manual", detail=detail)
|
||||
|
||||
|
||||
def unconfigured(item: Item, detail: str) -> VersionCheck:
|
||||
"""AVWatch knows how to track this item, but the capability it needs is
|
||||
not switched on.
|
||||
|
||||
Kept distinct from ``manual`` on purpose. "There is no firmware for this
|
||||
passive splitter" and "your API key is missing so 60 items could not be
|
||||
looked up" are completely different problems, and collapsing them into one
|
||||
'check by hand' list makes a configuration gap look like the gear was
|
||||
never tracked at all.
|
||||
"""
|
||||
return VersionCheck(item_id=item.id, status="unconfigured", detail=detail)
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Blackmagic Design.
|
||||
|
||||
BMD publishes their entire support catalogue as one JSON document, which makes
|
||||
them by far the best-behaved vendor in this fleet. One fetch resolves ATEM
|
||||
switchers and panels, Desktop Video (DeckLink), HyperDeck, the converter
|
||||
family and MultiView.
|
||||
|
||||
Verified structure of an entry:
|
||||
|
||||
{
|
||||
"name": "ATEM Switchers 10.4.1 Update",
|
||||
"date": "Last Thursday", # human, relative -- unusable
|
||||
"numericDate": 1788418810000, # ms epoch -- use this
|
||||
"desc": "This software update provides ...",
|
||||
"releaseNotesTitle": "Blackmagic ATEM Switchers 10.4.1",
|
||||
"relatedFamilies": ["atem-live-production-switchers"],
|
||||
"platforms": ["Mac OS X", "Windows"],
|
||||
"type": "Product Release",
|
||||
"urls": {"Windows": [{"major": 10, "minor": 4, "releaseNum": 1,
|
||||
"buildNum": 0, "beta": 255, "downloadId": "..."}]}
|
||||
}
|
||||
|
||||
``beta: 255`` marks a final release; anything lower is a beta build.
|
||||
|
||||
Inventory usage:
|
||||
|
||||
source:
|
||||
kind: blackmagic
|
||||
family: "ATEM Switchers" # matches the leading words of `name`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ..models import Item, VersionCheck
|
||||
from ..versions import parse
|
||||
from .base import Context, error
|
||||
|
||||
DOWNLOADS_URL = "https://www.blackmagicdesign.com/api/support/us/downloads.json"
|
||||
|
||||
# Family label -> regex capturing the version out of the `name` field.
|
||||
# Verified against the live feed on 2026-09-08.
|
||||
FAMILY_PATTERNS: dict[str, str] = {
|
||||
"ATEM Switchers": r"^ATEM Switchers\s+([\d.]+)\s+Update$",
|
||||
"Desktop Video": r"^Desktop Video\s+([\d.]+)$",
|
||||
"Blackmagic Converters": r"^Blackmagic Converters\s+([\d.]+)\s+Update$",
|
||||
"HyperDeck": r"^HyperDeck\s+([\d.]+)\s+Update$",
|
||||
"MultiView": r"^Blackmagic Multi\s?[Vv]iew\s+([\d.]+)",
|
||||
"Camera": r"^Blackmagic Camera\s+([\d.]+)",
|
||||
"Videohub": r"^Videohub\s+([\d.]+)",
|
||||
"Web Presenter": r"^Web Presenter\s+([\d.]+)",
|
||||
"DaVinci Resolve": r"^DaVinci Resolve\s+([\d.]+)",
|
||||
}
|
||||
|
||||
BETA_FINAL_SENTINEL = 255
|
||||
|
||||
|
||||
def _load_catalogue(ctx: Context) -> list[dict]:
|
||||
def fetch() -> list[dict]:
|
||||
# The catalogue is ~1.6 MB; a 6 hour TTL is plenty and keeps us off
|
||||
# their servers during multi-run days.
|
||||
resp = ctx.http.get(DOWNLOADS_URL, ttl=6 * 3600)
|
||||
if not resp.ok:
|
||||
raise RuntimeError(f"HTTP {resp.status} from BMD downloads feed")
|
||||
data = resp.json()
|
||||
entries = data.get("downloads")
|
||||
if not isinstance(entries, list):
|
||||
raise RuntimeError("unexpected BMD payload: no 'downloads' list")
|
||||
return entries
|
||||
|
||||
return ctx.memo("bmd:catalogue", fetch)
|
||||
|
||||
|
||||
def _is_beta(entry: dict) -> bool:
|
||||
for builds in (entry.get("urls") or {}).values():
|
||||
if builds and isinstance(builds, list):
|
||||
beta = builds[0].get("beta")
|
||||
if beta is not None and beta != BETA_FINAL_SENTINEL:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _released(entry: dict) -> datetime | None:
|
||||
ms = entry.get("numericDate")
|
||||
if not isinstance(ms, (int, float)):
|
||||
return None
|
||||
try:
|
||||
return datetime.fromtimestamp(ms / 1000.0, tz=timezone.utc)
|
||||
except (OverflowError, OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
class BlackmagicSource:
|
||||
kind = "blackmagic"
|
||||
|
||||
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||||
family = item.source.get("family")
|
||||
if not family:
|
||||
return error(item, "blackmagic source requires 'family'")
|
||||
|
||||
pattern = item.source.get("pattern") or FAMILY_PATTERNS.get(family)
|
||||
if not pattern:
|
||||
return error(
|
||||
item,
|
||||
f"unknown BMD family {family!r}; set an explicit 'pattern' "
|
||||
f"or use one of: {', '.join(sorted(FAMILY_PATTERNS))}",
|
||||
)
|
||||
|
||||
try:
|
||||
catalogue = _load_catalogue(ctx)
|
||||
except RuntimeError as exc:
|
||||
return error(item, str(exc))
|
||||
|
||||
rx = re.compile(pattern)
|
||||
include_beta = bool(item.source.get("include_beta", False))
|
||||
|
||||
best: tuple | None = None
|
||||
for entry in catalogue:
|
||||
name = (entry.get("name") or "").strip()
|
||||
m = rx.match(name)
|
||||
if not m:
|
||||
continue
|
||||
if not include_beta and _is_beta(entry):
|
||||
continue
|
||||
ver = parse(m.group(1))
|
||||
if not ver.valid:
|
||||
continue
|
||||
if best is None or ver > best[0]:
|
||||
best = (ver, entry)
|
||||
|
||||
if best is None:
|
||||
return error(item, f"no releases matched pattern for family {family!r}")
|
||||
|
||||
ver, entry = best
|
||||
notes = (entry.get("desc") or "").strip() or None
|
||||
title = entry.get("releaseNotesTitle")
|
||||
|
||||
return VersionCheck(
|
||||
item_id=item.id,
|
||||
latest=ver.raw,
|
||||
released=_released(entry),
|
||||
status="ok",
|
||||
release_notes=notes,
|
||||
release_url=item.check_url or "https://www.blackmagicdesign.com/support",
|
||||
detail=title,
|
||||
)
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Generic, config-driven sources: HTML scrape, JSON API, RSS/Atom, manual.
|
||||
|
||||
These carry the long tail of the fleet -- Shure, Crown, Novastar, Waves,
|
||||
Focusrite and friends publish version numbers on ordinary web pages with no
|
||||
API behind them. Everything is declared in inventory.yaml so a broken
|
||||
selector is a one-line YAML fix rather than a code change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html as html_mod
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
from ..models import Item, VersionCheck
|
||||
from ..versions import parse
|
||||
from .base import Context, error, manual
|
||||
|
||||
_TAG_RE = re.compile(r"<(script|style)\b.*?</\1>", re.S | re.I)
|
||||
_ANY_TAG = re.compile(r"<[^>]+>")
|
||||
|
||||
|
||||
def _visible_text(raw: str) -> str:
|
||||
text = _TAG_RE.sub(" ", raw)
|
||||
text = _ANY_TAG.sub(" ", text)
|
||||
text = html_mod.unescape(text)
|
||||
return re.sub(r"\s+", " ", text)
|
||||
|
||||
|
||||
class HtmlSource:
|
||||
"""Pull a version out of a web page with a regex.
|
||||
|
||||
Inventory usage:
|
||||
|
||||
source:
|
||||
kind: html
|
||||
url: https://www.resolume.com/download/
|
||||
pattern: 'Arena\\s+([\\d.]+)'
|
||||
strip_tags: true # search visible text rather than raw markup
|
||||
select: max # max (default) | first
|
||||
date_pattern: '([A-Z][a-z]+ \\d{1,2}, \\d{4})' # optional
|
||||
"""
|
||||
|
||||
kind = "html"
|
||||
|
||||
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||||
url = item.source.get("url")
|
||||
pattern = item.source.get("pattern")
|
||||
if not url or not pattern:
|
||||
return error(item, "html source requires 'url' and 'pattern'")
|
||||
|
||||
resp = ctx.http.get(url, ttl=item.source.get("ttl", 3600))
|
||||
if not resp.ok:
|
||||
return error(item, f"HTTP {resp.status} from {url}")
|
||||
|
||||
haystack = (
|
||||
_visible_text(resp.text)
|
||||
if item.source.get("strip_tags", True)
|
||||
else resp.text
|
||||
)
|
||||
|
||||
try:
|
||||
rx = re.compile(pattern, re.I)
|
||||
except re.error as exc:
|
||||
return error(item, f"bad 'pattern' regex: {exc}")
|
||||
|
||||
found = [m.group(1) if m.groups() else m.group(0) for m in rx.finditer(haystack)]
|
||||
versions = [p for p in (parse(f) for f in found) if p.valid]
|
||||
if not versions:
|
||||
return error(
|
||||
item,
|
||||
f"pattern matched nothing on {url} -- the page layout probably changed",
|
||||
)
|
||||
|
||||
chosen = (
|
||||
max(versions) if item.source.get("select", "max") == "max" else versions[0]
|
||||
)
|
||||
|
||||
released = None
|
||||
dpat = item.source.get("date_pattern")
|
||||
if dpat:
|
||||
dm = re.search(dpat, haystack)
|
||||
if dm:
|
||||
released = _try_dates(dm.group(1))
|
||||
|
||||
return VersionCheck(
|
||||
item_id=item.id,
|
||||
latest=chosen.raw,
|
||||
released=released,
|
||||
status="ok",
|
||||
release_url=item.check_url or url,
|
||||
detail=f"scraped from {url}",
|
||||
)
|
||||
|
||||
|
||||
def _dig(obj, path: str):
|
||||
"""Walk a dotted path with optional [n] indices: 'data.releases[0].version'."""
|
||||
cur = obj
|
||||
for token in path.split("."):
|
||||
if not token:
|
||||
continue
|
||||
m = re.match(r"^([^\[\]]*)((?:\[\d+\])*)$", token)
|
||||
if not m:
|
||||
return None
|
||||
key, idxs = m.group(1), m.group(2)
|
||||
if key:
|
||||
if not isinstance(cur, dict) or key not in cur:
|
||||
return None
|
||||
cur = cur[key]
|
||||
for i in re.findall(r"\[(\d+)\]", idxs):
|
||||
if not isinstance(cur, list) or int(i) >= len(cur):
|
||||
return None
|
||||
cur = cur[int(i)]
|
||||
return cur
|
||||
|
||||
|
||||
class JsonSource:
|
||||
"""Read a version from a JSON endpoint.
|
||||
|
||||
Inventory usage:
|
||||
|
||||
source:
|
||||
kind: json
|
||||
url: https://example.com/api/latest.json
|
||||
version_path: data.latest.version
|
||||
date_path: data.latest.released # optional
|
||||
notes_path: data.latest.notes # optional
|
||||
"""
|
||||
|
||||
kind = "json"
|
||||
|
||||
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||||
url = item.source.get("url")
|
||||
vpath = item.source.get("version_path")
|
||||
if not url or not vpath:
|
||||
return error(item, "json source requires 'url' and 'version_path'")
|
||||
|
||||
resp = ctx.http.get(url, ttl=item.source.get("ttl", 3600))
|
||||
if not resp.ok:
|
||||
return error(item, f"HTTP {resp.status} from {url}")
|
||||
try:
|
||||
data = resp.json()
|
||||
except ValueError as exc:
|
||||
return error(item, f"invalid JSON from {url}: {exc}")
|
||||
|
||||
raw_version = _dig(data, vpath)
|
||||
if raw_version is None:
|
||||
return error(item, f"version_path {vpath!r} not present in payload")
|
||||
ver = parse(str(raw_version))
|
||||
if not ver.valid:
|
||||
return error(item, f"unparseable version {raw_version!r}")
|
||||
|
||||
released = None
|
||||
if dpath := item.source.get("date_path"):
|
||||
released = _try_dates(_dig(data, dpath))
|
||||
notes = None
|
||||
if npath := item.source.get("notes_path"):
|
||||
val = _dig(data, npath)
|
||||
notes = str(val)[:4000] if val else None
|
||||
|
||||
return VersionCheck(
|
||||
item_id=item.id,
|
||||
latest=ver.raw,
|
||||
released=released,
|
||||
status="ok",
|
||||
release_notes=notes,
|
||||
release_url=item.check_url or url,
|
||||
detail=f"json:{vpath}",
|
||||
)
|
||||
|
||||
|
||||
class RssSource:
|
||||
"""Version from the newest entry title in an RSS or Atom feed.
|
||||
|
||||
Inventory usage:
|
||||
|
||||
source:
|
||||
kind: rss
|
||||
url: https://example.com/releases.atom
|
||||
pattern: '([\\d.]+)'
|
||||
"""
|
||||
|
||||
kind = "rss"
|
||||
|
||||
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||||
url = item.source.get("url")
|
||||
if not url:
|
||||
return error(item, "rss source requires 'url'")
|
||||
|
||||
resp = ctx.http.get(url, ttl=item.source.get("ttl", 1800))
|
||||
if not resp.ok:
|
||||
return error(item, f"HTTP {resp.status} from {url}")
|
||||
|
||||
try:
|
||||
root = ET.fromstring(resp.text.encode("utf-8", "replace"))
|
||||
except ET.ParseError as exc:
|
||||
return error(item, f"malformed feed: {exc}")
|
||||
|
||||
entries = _feed_entries(root)
|
||||
if not entries:
|
||||
return error(item, "feed contained no items")
|
||||
|
||||
rx = re.compile(item.source.get("pattern", r"([0-9]+(?:\.[0-9]+)+)"))
|
||||
for title, link, published in entries:
|
||||
m = rx.search(title or "")
|
||||
if not m:
|
||||
continue
|
||||
ver = parse(m.group(1) if m.groups() else m.group(0))
|
||||
if not ver.valid:
|
||||
continue
|
||||
return VersionCheck(
|
||||
item_id=item.id,
|
||||
latest=ver.raw,
|
||||
released=published,
|
||||
status="ok",
|
||||
release_url=link or item.check_url or url,
|
||||
detail=title,
|
||||
)
|
||||
|
||||
return error(item, "no feed entry title matched the version pattern")
|
||||
|
||||
|
||||
def _feed_entries(root) -> list[tuple[str | None, str | None, datetime | None]]:
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
out: list[tuple[str | None, str | None, datetime | None]] = []
|
||||
|
||||
for it in root.iter():
|
||||
tag = it.tag.split("}")[-1]
|
||||
if tag not in ("item", "entry"):
|
||||
continue
|
||||
title = _child_text(it, ("title",))
|
||||
link = _child_text(it, ("link", "guid")) or _atom_link(it, ns)
|
||||
date_text = _child_text(it, ("pubDate", "updated", "published", "date"))
|
||||
out.append((title, link, _try_dates(date_text)))
|
||||
return out
|
||||
|
||||
|
||||
def _child_text(el, names: tuple[str, ...]) -> str | None:
|
||||
for child in el:
|
||||
if child.tag.split("}")[-1] in names and (child.text or "").strip():
|
||||
return child.text.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _atom_link(el, ns) -> str | None:
|
||||
for child in el:
|
||||
if child.tag.split("}")[-1] == "link":
|
||||
href = child.attrib.get("href")
|
||||
if href:
|
||||
return href
|
||||
return None
|
||||
|
||||
|
||||
_DATE_FORMATS = (
|
||||
"%Y-%m-%d", "%d %b %Y", "%d %B %Y", "%b %d, %Y", "%B %d, %Y",
|
||||
"%m/%d/%Y", "%Y/%m/%d", "%Y-%m-%dT%H:%M:%S",
|
||||
)
|
||||
|
||||
|
||||
def _try_dates(text) -> datetime | None:
|
||||
if text is None:
|
||||
return None
|
||||
if isinstance(text, (int, float)):
|
||||
# Heuristic: values that large are milliseconds.
|
||||
secs = text / 1000.0 if text > 1e11 else float(text)
|
||||
try:
|
||||
return datetime.fromtimestamp(secs, tz=timezone.utc)
|
||||
except (OverflowError, OSError, ValueError):
|
||||
return None
|
||||
text = str(text).strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
dt = parsedate_to_datetime(text)
|
||||
if dt:
|
||||
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
for fmt in _DATE_FORMATS:
|
||||
try:
|
||||
return datetime.strptime(text, fmt).replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
class ManualSource:
|
||||
"""No machine-readable feed exists (or the vendor blocks scraping).
|
||||
|
||||
avwatch still tracks these: it runs the community risk scan, records what
|
||||
you tell it via ``avwatch set-installed``, and puts the item in a
|
||||
"verify by hand" section of the report with a deep link. That is a
|
||||
deliberate design choice -- a wrong scraped version is worse than an
|
||||
honest "go look".
|
||||
"""
|
||||
|
||||
kind = "manual"
|
||||
|
||||
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||||
note = item.source.get("note") or (
|
||||
"Vendor publishes no machine-readable version feed. Check the link."
|
||||
)
|
||||
return manual(item, note)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""GitHub Releases.
|
||||
|
||||
Covers the open-source half of the fleet: OBS, Bitfocus Companion and
|
||||
Satellite, RustDesk. Unauthenticated GitHub allows 60 requests/hour per IP,
|
||||
which is enough for a handful of items but not for issue scanning as well --
|
||||
set AVWATCH_GITHUB_TOKEN (a classic token with no scopes is fine) to lift the
|
||||
ceiling to 5000/hour.
|
||||
|
||||
Inventory usage:
|
||||
|
||||
source:
|
||||
kind: github
|
||||
repo: obsproject/obs-studio
|
||||
allow_prerelease: false
|
||||
tag_pattern: '^(?:v)?([\\d.]+)$' # optional override
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ..models import Item, VersionCheck
|
||||
from ..versions import parse
|
||||
from .base import Context, error
|
||||
|
||||
API = "https://api.github.com"
|
||||
DEFAULT_TAG_PATTERN = r"^(?:v|release-|Release\s*)?([0-9]+(?:\.[0-9]+)*)"
|
||||
|
||||
|
||||
def auth_headers() -> dict:
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
token = os.environ.get("AVWATCH_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def _parse_ts(text: str | None) -> datetime | None:
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(timezone.utc)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
class GithubSource:
|
||||
kind = "github"
|
||||
|
||||
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||||
repo = item.source.get("repo") or item.github_repo
|
||||
if not repo:
|
||||
return error(item, "github source requires 'repo' (owner/name)")
|
||||
|
||||
allow_pre = bool(item.source.get("allow_prerelease", False))
|
||||
rx = re.compile(item.source.get("tag_pattern") or DEFAULT_TAG_PATTERN)
|
||||
|
||||
# /releases (plural) so we can skip drafts and prereleases ourselves;
|
||||
# /releases/latest already excludes prereleases but also hides the
|
||||
# release notes of anything newer that is still flagged pre.
|
||||
resp = ctx.http.get(
|
||||
f"{API}/repos/{repo}/releases?per_page=20",
|
||||
headers=auth_headers(),
|
||||
ttl=3600,
|
||||
)
|
||||
if not resp.ok:
|
||||
if resp.status == 403:
|
||||
return error(item, "GitHub rate limit hit; set AVWATCH_GITHUB_TOKEN")
|
||||
if resp.status == 404:
|
||||
return error(item, f"repo not found: {repo}")
|
||||
return error(item, f"HTTP {resp.status} from GitHub for {repo}")
|
||||
|
||||
try:
|
||||
releases = resp.json()
|
||||
except ValueError as exc:
|
||||
return error(item, f"bad JSON from GitHub: {exc}")
|
||||
if not isinstance(releases, list) or not releases:
|
||||
return error(item, f"no releases published for {repo}")
|
||||
|
||||
best: tuple | None = None
|
||||
for rel in releases:
|
||||
if rel.get("draft"):
|
||||
continue
|
||||
if rel.get("prerelease") and not allow_pre:
|
||||
continue
|
||||
tag = (rel.get("tag_name") or "").strip()
|
||||
m = rx.match(tag)
|
||||
candidate = m.group(1) if m else tag
|
||||
ver = parse(candidate)
|
||||
if not ver.valid:
|
||||
continue
|
||||
if best is None or ver > best[0]:
|
||||
best = (ver, rel)
|
||||
|
||||
if best is None:
|
||||
return error(item, f"no parseable release tags for {repo}")
|
||||
|
||||
ver, rel = best
|
||||
body = (rel.get("body") or "").strip()
|
||||
if len(body) > 4000:
|
||||
body = body[:4000] + "\n...[truncated]"
|
||||
|
||||
return VersionCheck(
|
||||
item_id=item.id,
|
||||
latest=ver.raw,
|
||||
released=_parse_ts(rel.get("published_at") or rel.get("created_at")),
|
||||
status="ok",
|
||||
release_notes=body or None,
|
||||
release_url=rel.get("html_url"),
|
||||
detail=rel.get("name") or rel.get("tag_name"),
|
||||
)
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Claude-with-web-search version lookup.
|
||||
|
||||
Why this exists: almost every AV vendor's support site is now a
|
||||
JavaScript-rendered single-page app. Verified on 2026-09-08 -- Shure,
|
||||
Resolume, Focusrite, Teradek, Line 6, Novastar and Rational Acoustics all
|
||||
return HTTP 200 with essentially no version text in the HTML. A regex
|
||||
scraper cannot read them, and a headless browser for 70 vendors is a
|
||||
maintenance sink.
|
||||
|
||||
So for that long tail, Claude does the lookup with the server-side web search
|
||||
tool and reports the version plus the page it came from. This is a factual
|
||||
retrieval job rather than a judgement call, so it runs on a cheaper model than
|
||||
the verdict layer by default.
|
||||
|
||||
Guardrails, because a confidently wrong version number is worse than none:
|
||||
* The model must return a source URL, and must say "unknown" when unsure.
|
||||
* Answers are cached in SQLite (default 6 days), so the Monday digest pays
|
||||
for the lookup and mid-week scans reuse it.
|
||||
* The result is labelled in the report as web-sourced, not vendor-verified.
|
||||
|
||||
Inventory usage:
|
||||
|
||||
source:
|
||||
kind: websearch
|
||||
query: "Shure Axient Digital AD4Q receiver firmware"
|
||||
hint: "Firmware is distributed through Shure Update Utility."
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ..models import Item, VersionCheck
|
||||
from ..versions import parse
|
||||
from .base import Context, error, manual, unconfigured
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SYSTEM = """\
|
||||
You look up the current released firmware or software version for professional \
|
||||
audio/video production equipment, using web search.
|
||||
|
||||
Return ONLY a JSON object, no prose around it:
|
||||
|
||||
{
|
||||
"version": "<the version string exactly as the vendor writes it, or null>",
|
||||
"released": "<YYYY-MM-DD if you can establish it, else null>",
|
||||
"source_url": "<the page you took it from, or null>",
|
||||
"confidence": "high" | "medium" | "low",
|
||||
"notes": "<one short sentence: caveats, or what a human should verify>"
|
||||
}
|
||||
|
||||
Rules that matter more than being helpful:
|
||||
* Report the current GENERAL RELEASE version. Not a beta, not a release \
|
||||
candidate, not a rumour, not a version discussed on a forum as upcoming.
|
||||
* The vendor's own download or support page is the only authoritative \
|
||||
source. A forum post or reseller listing is at best "low" confidence.
|
||||
* If you cannot find a version you are confident is current, set version to \
|
||||
null and confidence to "low" and say why in notes. That is a correct answer. \
|
||||
A wrong version number here causes a production team to make a bad decision, \
|
||||
so never guess or interpolate.
|
||||
* Many products share one firmware package across a family. If the version \
|
||||
you find covers a family rather than this exact model, say so in notes.
|
||||
* Do not report the version of a different product with a similar name.
|
||||
"""
|
||||
|
||||
|
||||
def _extract_json(text: str) -> dict | None:
|
||||
"""Tolerant JSON extraction -- the model may wrap the object in prose or a
|
||||
code fence despite instructions."""
|
||||
text = text.strip()
|
||||
fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.S)
|
||||
if fence:
|
||||
text = fence.group(1)
|
||||
else:
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start == -1 or end <= start:
|
||||
return None
|
||||
text = text[start:end + 1]
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError:
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
class WebSearchSource:
|
||||
kind = "websearch"
|
||||
|
||||
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||||
cfg = ctx.scratch.get("lookup_cfg")
|
||||
store = ctx.scratch.get("store")
|
||||
|
||||
if cfg is None or not getattr(cfg, "enabled", False):
|
||||
return unconfigured(
|
||||
item,
|
||||
"Version lookup is switched off for this run "
|
||||
"(--no-llm, offline, or lookup.enabled = false).",
|
||||
)
|
||||
if not getattr(cfg, "api_key", ""):
|
||||
return unconfigured(
|
||||
item,
|
||||
"Needs ANTHROPIC_API_KEY to look up the current version.",
|
||||
)
|
||||
|
||||
# 1. cache
|
||||
if store is not None:
|
||||
cached = store.get_lookup(item.id, cfg.ttl_days)
|
||||
if cached:
|
||||
return self._from_cache(item, cached)
|
||||
|
||||
# 2. budget for this run
|
||||
spent = ctx.scratch.get("lookup_calls", 0)
|
||||
if spent >= cfg.max_items_per_run:
|
||||
return unconfigured(
|
||||
item,
|
||||
f"Lookup budget for this run is used up "
|
||||
f"(lookup.max_items_per_run = {cfg.max_items_per_run}). "
|
||||
f"This item will be picked up on the next run.",
|
||||
)
|
||||
|
||||
client = self._client(cfg, ctx)
|
||||
if client is None:
|
||||
return unconfigured(
|
||||
item,
|
||||
"The 'anthropic' package is not installed; "
|
||||
"run: pip install anthropic",
|
||||
)
|
||||
|
||||
query = item.source.get("query") or f"{item.vendor} {item.name}"
|
||||
hint = item.source.get("hint") or ""
|
||||
prompt = (
|
||||
f"What is the current released firmware/software version for:\n\n"
|
||||
f" Product: {item.name}\n"
|
||||
f" Vendor: {item.vendor}\n"
|
||||
f" Search: {query}\n"
|
||||
+ (f" Context: {hint}\n" if hint else "")
|
||||
+ "\nSearch the web, then answer with the JSON object only."
|
||||
)
|
||||
|
||||
try:
|
||||
resp = client.messages.create(
|
||||
model=cfg.model,
|
||||
max_tokens=1200,
|
||||
system=[{
|
||||
"type": "text",
|
||||
"text": SYSTEM,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}],
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
tools=[{
|
||||
"type": "web_search_20260209",
|
||||
"name": "web_search",
|
||||
"max_uses": cfg.max_searches_per_item,
|
||||
}],
|
||||
)
|
||||
ctx.scratch["lookup_calls"] = spent + 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("web-search lookup failed for %s: %s", item.id, exc)
|
||||
return error(item, f"web-search lookup failed: {type(exc).__name__}")
|
||||
|
||||
if getattr(resp, "stop_reason", None) == "refusal":
|
||||
return manual(item, "Web-search lookup was declined; check by hand.")
|
||||
|
||||
text = "\n".join(b.text for b in resp.content if b.type == "text")
|
||||
data = _extract_json(text)
|
||||
if data is None:
|
||||
return error(item, "web-search lookup returned no parseable JSON")
|
||||
|
||||
version = data.get("version")
|
||||
version = str(version).strip() if version else None
|
||||
released = data.get("released")
|
||||
source_url = data.get("source_url")
|
||||
confidence = str(data.get("confidence", "low")).lower()
|
||||
notes = str(data.get("notes", "")).strip() or None
|
||||
|
||||
if store is not None:
|
||||
store.put_lookup(item.id, version, released, source_url,
|
||||
confidence, notes)
|
||||
|
||||
return self._build(item, version, released, source_url, confidence,
|
||||
notes, cached_age=None)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _client(self, cfg, ctx: Context):
|
||||
if "lookup_client" in ctx.scratch:
|
||||
return ctx.scratch["lookup_client"]
|
||||
try:
|
||||
import anthropic
|
||||
except ImportError:
|
||||
ctx.scratch["lookup_client"] = None
|
||||
return None
|
||||
client = anthropic.Anthropic(api_key=cfg.api_key, timeout=180.0)
|
||||
ctx.scratch["lookup_client"] = client
|
||||
return client
|
||||
|
||||
def _from_cache(self, item: Item, cached: dict) -> VersionCheck:
|
||||
return self._build(
|
||||
item,
|
||||
cached.get("version"),
|
||||
cached.get("released"),
|
||||
cached.get("source_url"),
|
||||
cached.get("confidence"),
|
||||
cached.get("notes"),
|
||||
cached_age=cached.get("age_days"),
|
||||
)
|
||||
|
||||
def _build(self, item: Item, version, released, source_url, confidence,
|
||||
notes, cached_age) -> VersionCheck:
|
||||
if not version or str(version).lower() in ("null", "none", "unknown"):
|
||||
reason = notes or "No current version could be established."
|
||||
return manual(item, f"Web lookup could not confirm a version. {reason}")
|
||||
|
||||
ver = parse(str(version))
|
||||
if not ver.valid:
|
||||
return manual(
|
||||
item,
|
||||
f"Web lookup returned {version!r}, which is not a version "
|
||||
f"number we can compare. {notes or ''}".strip(),
|
||||
)
|
||||
|
||||
released_dt = None
|
||||
if released:
|
||||
try:
|
||||
released_dt = datetime.fromisoformat(
|
||||
str(released).replace("Z", "+00:00")
|
||||
)
|
||||
if released_dt.tzinfo is None:
|
||||
released_dt = released_dt.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
released_dt = None
|
||||
|
||||
# Low confidence is not good enough to drive an install decision.
|
||||
if str(confidence).lower() == "low":
|
||||
return manual(
|
||||
item,
|
||||
f"Web lookup suggests {version} but is not confident. "
|
||||
f"{notes or ''} Verify on the vendor page before acting.".strip(),
|
||||
)
|
||||
|
||||
bits = [f"web lookup, {confidence} confidence"]
|
||||
if cached_age is not None:
|
||||
bits.append(f"cached {cached_age:.1f}d")
|
||||
if notes:
|
||||
bits.append(notes)
|
||||
|
||||
return VersionCheck(
|
||||
item_id=item.id,
|
||||
latest=ver.raw,
|
||||
released=released_dt,
|
||||
status="ok",
|
||||
release_notes=notes,
|
||||
release_url=source_url or item.check_url,
|
||||
detail=" | ".join(bits),
|
||||
)
|
||||
Reference in New Issue
Block a user