peter
b59fd38a9c
The log now has a stored, queryable copy for the diagnostics console: every WARNING and above plus the request line of every request, kept for log_retention_days (default 7, no "forever") and pruned hourly by the new log_gc job. Rows are built on the logging thread from the same redacted fields as the stdout line, queued without blocking and written in batches by one background thread on its own connection. A full queue or a failed write drops and reports instead of retrying. Flood control keeps 30 of any one message per minute plus a count of the rest; anonymous request lines are capped at 60 a minute per client, and the client address is never stored. Deleting an account deletes its stored events. Measured on prod first: a YouTube RSS outage wrote 11k warnings in four hours, one per channel per poll. The poll now writes one warning with per-reason counts and a few examples, and a failure on our side gets one traceback per poll instead of one per channel.
64 lines
2.4 KiB
Python
64 lines
2.4 KiB
Python
"""Free per-channel RSS feed reader (no API quota). Returns the channel's most recent
|
|
~15 uploads as lightweight stubs for fast fresh-video detection."""
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
import feedparser
|
|
import httpx
|
|
|
|
RSS_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
|
|
|
|
|
|
class RssError(Exception):
|
|
"""The feed could not be read (transport failure or a non-200 answer).
|
|
|
|
Deliberately NOT the same as an empty feed: a failed poll used to return `[]`, which the
|
|
caller stamped as a successful `last_rss_at` — so a channel whose feed 404s permanently
|
|
(deleted, renamed id) looked like it was being polled fine forever, and nothing surfaced it.
|
|
|
|
`reason` is a short stable code (`http_404`, `transport`) so a poll over many channels can report
|
|
WHAT failed as counts instead of one line per channel."""
|
|
|
|
def __init__(self, message: str, reason: str):
|
|
super().__init__(message)
|
|
self.reason = reason
|
|
|
|
|
|
def fetch_channel_feed(channel_id: str) -> list[dict]:
|
|
"""The channel's recent uploads as stubs. `[]` means the feed really is empty; a failure
|
|
raises RssError so the caller can leave the channel un-stamped and log it."""
|
|
url = RSS_URL.format(channel_id=channel_id)
|
|
try:
|
|
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"})
|
|
except httpx.HTTPError as exc:
|
|
raise RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport") from exc
|
|
if resp.status_code != 200:
|
|
raise RssError(
|
|
f"RSS fetch for {channel_id} returned HTTP {resp.status_code}",
|
|
f"http_{resp.status_code}",
|
|
)
|
|
|
|
parsed = feedparser.parse(resp.content)
|
|
out: list[dict] = []
|
|
for entry in parsed.entries:
|
|
video_id = entry.get("yt_videoid")
|
|
if not video_id:
|
|
continue
|
|
published = None
|
|
if entry.get("published_parsed"):
|
|
published = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc)
|
|
thumbnail = None
|
|
media = entry.get("media_thumbnail") or []
|
|
if media:
|
|
thumbnail = media[0].get("url")
|
|
out.append(
|
|
{
|
|
"id": video_id,
|
|
"channel_id": channel_id,
|
|
"title": entry.get("title"),
|
|
"published_at": published,
|
|
"thumbnail_url": thumbnail,
|
|
}
|
|
)
|
|
return out
|