311 lines
9.6 KiB
Python
311 lines
9.6 KiB
Python
|
"""Generic, config-driven sources: HTML scrape, JSON API, RSS/Atom, manual.
|
||
|
|
|
||
|
|
These carry the long tail of the fleet -- Shure, Crown, Novastar, Waves,
|
||
|
|
Focusrite and friends publish version numbers on ordinary web pages with no
|
||
|
|
API behind them. Everything is declared in inventory.yaml so a broken
|
||
|
|
selector is a one-line YAML fix rather than a code change.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import html as html_mod
|
||
|
|
import re
|
||
|
|
import xml.etree.ElementTree as ET
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
from email.utils import parsedate_to_datetime
|
||
|
|
|
||
|
|
from ..models import Item, VersionCheck
|
||
|
|
from ..versions import parse
|
||
|
|
from .base import Context, error, manual
|
||
|
|
|
||
|
|
_TAG_RE = re.compile(r"<(script|style)\b.*?</\1>", re.S | re.I)
|
||
|
|
_ANY_TAG = re.compile(r"<[^>]+>")
|
||
|
|
|
||
|
|
|
||
|
|
def _visible_text(raw: str) -> str:
|
||
|
|
text = _TAG_RE.sub(" ", raw)
|
||
|
|
text = _ANY_TAG.sub(" ", text)
|
||
|
|
text = html_mod.unescape(text)
|
||
|
|
return re.sub(r"\s+", " ", text)
|
||
|
|
|
||
|
|
|
||
|
|
class HtmlSource:
|
||
|
|
"""Pull a version out of a web page with a regex.
|
||
|
|
|
||
|
|
Inventory usage:
|
||
|
|
|
||
|
|
source:
|
||
|
|
kind: html
|
||
|
|
url: https://www.resolume.com/download/
|
||
|
|
pattern: 'Arena\\s+([\\d.]+)'
|
||
|
|
strip_tags: true # search visible text rather than raw markup
|
||
|
|
select: max # max (default) | first
|
||
|
|
date_pattern: '([A-Z][a-z]+ \\d{1,2}, \\d{4})' # optional
|
||
|
|
"""
|
||
|
|
|
||
|
|
kind = "html"
|
||
|
|
|
||
|
|
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||
|
|
url = item.source.get("url")
|
||
|
|
pattern = item.source.get("pattern")
|
||
|
|
if not url or not pattern:
|
||
|
|
return error(item, "html source requires 'url' and 'pattern'")
|
||
|
|
|
||
|
|
resp = ctx.http.get(url, ttl=item.source.get("ttl", 3600))
|
||
|
|
if not resp.ok:
|
||
|
|
return error(item, f"HTTP {resp.status} from {url}")
|
||
|
|
|
||
|
|
haystack = (
|
||
|
|
_visible_text(resp.text)
|
||
|
|
if item.source.get("strip_tags", True)
|
||
|
|
else resp.text
|
||
|
|
)
|
||
|
|
|
||
|
|
try:
|
||
|
|
rx = re.compile(pattern, re.I)
|
||
|
|
except re.error as exc:
|
||
|
|
return error(item, f"bad 'pattern' regex: {exc}")
|
||
|
|
|
||
|
|
found = [m.group(1) if m.groups() else m.group(0) for m in rx.finditer(haystack)]
|
||
|
|
versions = [p for p in (parse(f) for f in found) if p.valid]
|
||
|
|
if not versions:
|
||
|
|
return error(
|
||
|
|
item,
|
||
|
|
f"pattern matched nothing on {url} -- the page layout probably changed",
|
||
|
|
)
|
||
|
|
|
||
|
|
chosen = (
|
||
|
|
max(versions) if item.source.get("select", "max") == "max" else versions[0]
|
||
|
|
)
|
||
|
|
|
||
|
|
released = None
|
||
|
|
dpat = item.source.get("date_pattern")
|
||
|
|
if dpat:
|
||
|
|
dm = re.search(dpat, haystack)
|
||
|
|
if dm:
|
||
|
|
released = _try_dates(dm.group(1))
|
||
|
|
|
||
|
|
return VersionCheck(
|
||
|
|
item_id=item.id,
|
||
|
|
latest=chosen.raw,
|
||
|
|
released=released,
|
||
|
|
status="ok",
|
||
|
|
release_url=item.check_url or url,
|
||
|
|
detail=f"scraped from {url}",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _dig(obj, path: str):
|
||
|
|
"""Walk a dotted path with optional [n] indices: 'data.releases[0].version'."""
|
||
|
|
cur = obj
|
||
|
|
for token in path.split("."):
|
||
|
|
if not token:
|
||
|
|
continue
|
||
|
|
m = re.match(r"^([^\[\]]*)((?:\[\d+\])*)$", token)
|
||
|
|
if not m:
|
||
|
|
return None
|
||
|
|
key, idxs = m.group(1), m.group(2)
|
||
|
|
if key:
|
||
|
|
if not isinstance(cur, dict) or key not in cur:
|
||
|
|
return None
|
||
|
|
cur = cur[key]
|
||
|
|
for i in re.findall(r"\[(\d+)\]", idxs):
|
||
|
|
if not isinstance(cur, list) or int(i) >= len(cur):
|
||
|
|
return None
|
||
|
|
cur = cur[int(i)]
|
||
|
|
return cur
|
||
|
|
|
||
|
|
|
||
|
|
class JsonSource:
|
||
|
|
"""Read a version from a JSON endpoint.
|
||
|
|
|
||
|
|
Inventory usage:
|
||
|
|
|
||
|
|
source:
|
||
|
|
kind: json
|
||
|
|
url: https://example.com/api/latest.json
|
||
|
|
version_path: data.latest.version
|
||
|
|
date_path: data.latest.released # optional
|
||
|
|
notes_path: data.latest.notes # optional
|
||
|
|
"""
|
||
|
|
|
||
|
|
kind = "json"
|
||
|
|
|
||
|
|
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||
|
|
url = item.source.get("url")
|
||
|
|
vpath = item.source.get("version_path")
|
||
|
|
if not url or not vpath:
|
||
|
|
return error(item, "json source requires 'url' and 'version_path'")
|
||
|
|
|
||
|
|
resp = ctx.http.get(url, ttl=item.source.get("ttl", 3600))
|
||
|
|
if not resp.ok:
|
||
|
|
return error(item, f"HTTP {resp.status} from {url}")
|
||
|
|
try:
|
||
|
|
data = resp.json()
|
||
|
|
except ValueError as exc:
|
||
|
|
return error(item, f"invalid JSON from {url}: {exc}")
|
||
|
|
|
||
|
|
raw_version = _dig(data, vpath)
|
||
|
|
if raw_version is None:
|
||
|
|
return error(item, f"version_path {vpath!r} not present in payload")
|
||
|
|
ver = parse(str(raw_version))
|
||
|
|
if not ver.valid:
|
||
|
|
return error(item, f"unparseable version {raw_version!r}")
|
||
|
|
|
||
|
|
released = None
|
||
|
|
if dpath := item.source.get("date_path"):
|
||
|
|
released = _try_dates(_dig(data, dpath))
|
||
|
|
notes = None
|
||
|
|
if npath := item.source.get("notes_path"):
|
||
|
|
val = _dig(data, npath)
|
||
|
|
notes = str(val)[:4000] if val else None
|
||
|
|
|
||
|
|
return VersionCheck(
|
||
|
|
item_id=item.id,
|
||
|
|
latest=ver.raw,
|
||
|
|
released=released,
|
||
|
|
status="ok",
|
||
|
|
release_notes=notes,
|
||
|
|
release_url=item.check_url or url,
|
||
|
|
detail=f"json:{vpath}",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class RssSource:
|
||
|
|
"""Version from the newest entry title in an RSS or Atom feed.
|
||
|
|
|
||
|
|
Inventory usage:
|
||
|
|
|
||
|
|
source:
|
||
|
|
kind: rss
|
||
|
|
url: https://example.com/releases.atom
|
||
|
|
pattern: '([\\d.]+)'
|
||
|
|
"""
|
||
|
|
|
||
|
|
kind = "rss"
|
||
|
|
|
||
|
|
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||
|
|
url = item.source.get("url")
|
||
|
|
if not url:
|
||
|
|
return error(item, "rss source requires 'url'")
|
||
|
|
|
||
|
|
resp = ctx.http.get(url, ttl=item.source.get("ttl", 1800))
|
||
|
|
if not resp.ok:
|
||
|
|
return error(item, f"HTTP {resp.status} from {url}")
|
||
|
|
|
||
|
|
try:
|
||
|
|
root = ET.fromstring(resp.text.encode("utf-8", "replace"))
|
||
|
|
except ET.ParseError as exc:
|
||
|
|
return error(item, f"malformed feed: {exc}")
|
||
|
|
|
||
|
|
entries = _feed_entries(root)
|
||
|
|
if not entries:
|
||
|
|
return error(item, "feed contained no items")
|
||
|
|
|
||
|
|
rx = re.compile(item.source.get("pattern", r"([0-9]+(?:\.[0-9]+)+)"))
|
||
|
|
for title, link, published in entries:
|
||
|
|
m = rx.search(title or "")
|
||
|
|
if not m:
|
||
|
|
continue
|
||
|
|
ver = parse(m.group(1) if m.groups() else m.group(0))
|
||
|
|
if not ver.valid:
|
||
|
|
continue
|
||
|
|
return VersionCheck(
|
||
|
|
item_id=item.id,
|
||
|
|
latest=ver.raw,
|
||
|
|
released=published,
|
||
|
|
status="ok",
|
||
|
|
release_url=link or item.check_url or url,
|
||
|
|
detail=title,
|
||
|
|
)
|
||
|
|
|
||
|
|
return error(item, "no feed entry title matched the version pattern")
|
||
|
|
|
||
|
|
|
||
|
|
def _feed_entries(root) -> list[tuple[str | None, str | None, datetime | None]]:
|
||
|
|
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||
|
|
out: list[tuple[str | None, str | None, datetime | None]] = []
|
||
|
|
|
||
|
|
for it in root.iter():
|
||
|
|
tag = it.tag.split("}")[-1]
|
||
|
|
if tag not in ("item", "entry"):
|
||
|
|
continue
|
||
|
|
title = _child_text(it, ("title",))
|
||
|
|
link = _child_text(it, ("link", "guid")) or _atom_link(it, ns)
|
||
|
|
date_text = _child_text(it, ("pubDate", "updated", "published", "date"))
|
||
|
|
out.append((title, link, _try_dates(date_text)))
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def _child_text(el, names: tuple[str, ...]) -> str | None:
|
||
|
|
for child in el:
|
||
|
|
if child.tag.split("}")[-1] in names and (child.text or "").strip():
|
||
|
|
return child.text.strip()
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _atom_link(el, ns) -> str | None:
|
||
|
|
for child in el:
|
||
|
|
if child.tag.split("}")[-1] == "link":
|
||
|
|
href = child.attrib.get("href")
|
||
|
|
if href:
|
||
|
|
return href
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
_DATE_FORMATS = (
|
||
|
|
"%Y-%m-%d", "%d %b %Y", "%d %B %Y", "%b %d, %Y", "%B %d, %Y",
|
||
|
|
"%m/%d/%Y", "%Y/%m/%d", "%Y-%m-%dT%H:%M:%S",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _try_dates(text) -> datetime | None:
|
||
|
|
if text is None:
|
||
|
|
return None
|
||
|
|
if isinstance(text, (int, float)):
|
||
|
|
# Heuristic: values that large are milliseconds.
|
||
|
|
secs = text / 1000.0 if text > 1e11 else float(text)
|
||
|
|
try:
|
||
|
|
return datetime.fromtimestamp(secs, tz=timezone.utc)
|
||
|
|
except (OverflowError, OSError, ValueError):
|
||
|
|
return None
|
||
|
|
text = str(text).strip()
|
||
|
|
if not text:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
dt = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||
|
|
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
||
|
|
except ValueError:
|
||
|
|
pass
|
||
|
|
try:
|
||
|
|
dt = parsedate_to_datetime(text)
|
||
|
|
if dt:
|
||
|
|
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
||
|
|
except (TypeError, ValueError):
|
||
|
|
pass
|
||
|
|
for fmt in _DATE_FORMATS:
|
||
|
|
try:
|
||
|
|
return datetime.strptime(text, fmt).replace(tzinfo=timezone.utc)
|
||
|
|
except ValueError:
|
||
|
|
continue
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
class ManualSource:
|
||
|
|
"""No machine-readable feed exists (or the vendor blocks scraping).
|
||
|
|
|
||
|
|
avwatch still tracks these: it runs the community risk scan, records what
|
||
|
|
you tell it via ``avwatch set-installed``, and puts the item in a
|
||
|
|
"verify by hand" section of the report with a deep link. That is a
|
||
|
|
deliberate design choice -- a wrong scraped version is worse than an
|
||
|
|
honest "go look".
|
||
|
|
"""
|
||
|
|
|
||
|
|
kind = "manual"
|
||
|
|
|
||
|
|
def check(self, item: Item, ctx: Context) -> VersionCheck:
|
||
|
|
note = item.source.get("note") or (
|
||
|
|
"Vendor publishes no machine-readable version feed. Check the link."
|
||
|
|
)
|
||
|
|
return manual(item, note)
|