From 29094f5f5113e6eb727d85c49cdb8346c70ea283 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 19 Sep 2026 04:51:31 +0200 Subject: [PATCH 01/21] fix(sync): read a dark RSS feed through the API instead YouTube's RSS goes dark per channel for hours (prod 2026-09-18/19: 393 of 396 feeds answered 404 for four hours, sticky per channel, so a retry does not help), and nothing else read a followed channel's new uploads. A channel whose feed has not answered for `rss_fallback_hours` (admin knob, default 2, 0 = off) now gets one page of its uploads playlist through the Data API (1 unit), at most once per window, yielding to the backfill reserve. Spend is attributed to its own quota action. --- .../0088_channels_feed_fallback_at.py | 30 +++++ backend/app/config.py | 3 + backend/app/models.py | 3 + backend/app/quota.py | 1 + backend/app/sync/runner.py | 98 ++++++++++++++- backend/app/sysconfig.py | 1 + backend/tests/test_feed_fallback.py | 115 ++++++++++++++++++ backend/tests/test_followed_channels.py | 2 +- backend/tests/test_sync_guards.py | 50 +++++++- frontend/src/i18n/locales/en/config.json | 4 + .../src/i18n/locales/en/quotaActions.json | 1 + frontend/src/i18n/locales/hu/config.json | 4 + .../src/i18n/locales/hu/quotaActions.json | 1 + 13 files changed, 302 insertions(+), 11 deletions(-) create mode 100644 backend/alembic/versions/0088_channels_feed_fallback_at.py create mode 100644 backend/tests/test_feed_fallback.py diff --git a/backend/alembic/versions/0088_channels_feed_fallback_at.py b/backend/alembic/versions/0088_channels_feed_fallback_at.py new file mode 100644 index 00000000..8680606c --- /dev/null +++ b/backend/alembic/versions/0088_channels_feed_fallback_at.py @@ -0,0 +1,30 @@ +"""add channels.feed_fallback_at — when a channel's uploads were last read through the API because +its RSS feed would not answer + +Revision ID: 0088_channels_feed_fallback_at +Revises: 0087_users_diagnostics_level +Create Date: 2026-09-19 + +YouTube's RSS feeds go dark for hours at a time, per channel (2026-09-18/19: 393 of 396 answered 404 +for four hours; the same channel answers 404 to every retry). The RSS poll then reads the newest +uploads through the Data API instead, at a quota cost, and at most once per `rss_fallback_hours` per +channel: this column is that clock. NULL = never needed it. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0088_channels_feed_fallback_at" +down_revision: str | None = "0087_users_diagnostics_level" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.add_column( + "channels", sa.Column("feed_fallback_at", sa.DateTime(timezone=True), nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("channels", "feed_fallback_at") diff --git a/backend/app/config.py b/backend/app/config.py index 2b2ad824..2490b51a 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -119,6 +119,9 @@ class Settings(BaseSettings): # Recent-first backfill: how far back to fetch on the first pass per channel. backfill_recent_max_videos: int = 100 backfill_recent_max_days: int = 365 + # A channel whose RSS feed has not answered for this many hours gets its newest uploads read + # through the Data API instead (1 unit), at most once per this many hours. 0 = never. + rss_fallback_hours: int = 2 # Shorts are confirmed by probing youtube.com/shorts/. Only videos at or below # this duration (and not livestreams) are probed; longer videos are never Shorts. diff --git a/backend/app/models.py b/backend/app/models.py index 173d4b8a..b4273184 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -241,6 +241,9 @@ class Channel(Base, TimestampMixin): details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + # When the newest uploads were last read through the Data API because the RSS feed would not + # answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock. + feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) backfill_cursor: Mapped[str | None] = mapped_column(String(128)) backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # True when this channel only entered the catalog because a live YouTube search diff --git a/backend/app/quota.py b/backend/app/quota.py index e630a34c..c7d91b38 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -37,6 +37,7 @@ class QuotaAction: PLAYLISTS_DELETE = "playlists_delete" VIDEOS_BACKFILL_RECENT = "videos_backfill_recent" VIDEOS_BACKFILL_FULL = "videos_backfill_full" + VIDEOS_FEED_FALLBACK = "videos_feed_fallback" VIDEOS_ENRICH = "videos_enrich" VIDEOS_LOOKUP = "videos_lookup" VIDEOS_SEARCH = "videos_search" diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index 7220620c..be7cef0b 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -9,8 +9,9 @@ import logging import threading from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timedelta, timezone -from sqlalchemy import select +from sqlalchemy import func, or_, select from sqlalchemy.orm import Session from app import logs, progress, quota, sysconfig @@ -25,11 +26,13 @@ from app.sync.videos import ( backfill_channel_deep, backfill_channel_recent, enrich_pending, + insert_stubs, reconcile_full_history, refresh_live, run_shorts_classification, + stub_from_playlist_item, ) -from app.youtube.client import YouTubeClient +from app.youtube.client import YouTubeClient, YouTubeError from app.youtube.rss import RssError, fetch_channel_feed # RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded. @@ -85,6 +88,9 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str # of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now. reasons: Counter[str] = Counter() examples: list[str] = [] + # Channels whose FEED would not answer (not those whose rows failed to apply: the API would + # hand the same rows to the same broken write) — the feed fallback's candidates. + unread: list[str] = [] def fail(reason: str, detail: str, *, unexpected: str | None = None) -> None: """Count a channel that did not get polled. `unexpected` is the message for a failure that @@ -120,6 +126,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str except RssError as exc: log.debug("RSS poll skipped for channel %s: %s", channel.id, exc) fail(exc.reason, f"{channel.id}: {exc}") + unread.append(channel.id) # silent-ok: `fail(unexpected=…)` is the report — it logs at ERROR with the # traceback, once per reason per poll, which is the flood control this loop needs. except Exception as exc: # noqa: BLE001 — one channel must not stop the poll @@ -143,6 +150,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str # One place that advances the counter, whatever happened above. done += 1 progress.report(done, total, "rss_poll") + fallback = run_feed_fallback(db, unread) if failed: log.warning( "RSS poll: %d/%d channels could not be polled", @@ -164,8 +172,90 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str # subscriptions "all channels failed" is also what ONE dead feed id looks like, and a red # card the user can't act on is its own kind of dishonest. `channel: HTTP 404` says # "fix this subscription"; `Connection refused` says "the egress is down". - raise RuntimeError(f"RSS poll failed for all {total} channels — first: {first_error}") - return {"new": new, "failed": failed} + # The fallback's count rides along: a red card during an RSS outage should still say + # whether the uploads are arriving another way. + raise RuntimeError( + f"RSS poll failed for all {total} channels — first: {first_error}" + f" (read through the API instead: {fallback['read']})" + ) + return {"new": new + fallback["new"], "failed": failed, "fallback": fallback["read"]} + + +def run_feed_fallback(db: Session, channel_ids: list[str]) -> dict[str, int]: + """Read the newest uploads through the Data API for the channels among `channel_ids` whose feed + has not answered for `rss_fallback_hours` — and that were not read this way within as long. + + YouTube's RSS goes dark per channel for hours at a time (2026-09-18/19: 393 of 396 feeds answered + 404 for four hours, and a retry of the same channel gets the same 404), and nothing else reads a + followed channel's new uploads: recent backfill runs once per channel, ever. One page of the + uploads playlist is 1 unit, so a whole day of total outage costs 12 units per channel at the + default 2 hours; it yields to the backfill reserve like every other background read. The clock is + `last_rss_at` or `feed_fallback_at`, whichever is later, so a channel is read at most once per + window whether its feed came back in between or not.""" + hours = sysconfig.get_int(db, "rss_fallback_hours") + if hours <= 0 or not channel_ids: + return {"read": 0, "new": 0, "failed": 0} + now = datetime.now(timezone.utc) + last_read = func.greatest(Channel.last_rss_at, Channel.feed_fallback_at) + due = ( + db.execute( + select(Channel) + .where( + Channel.id.in_(channel_ids), + Channel.uploads_playlist_id.is_not(None), + or_(last_read.is_(None), last_read < now - timedelta(hours=hours)), + ) + .order_by(Channel.id) + ) + .scalars() + .all() + ) + user = get_service_user(db) if due else None + if user is None: + return {"read": 0, "new": 0, "failed": 0} + read = new = failed = 0 + with ( + quota.attribute(None, quota.QuotaAction.VIDEOS_FEED_FALLBACK), + YouTubeClient(db, user) as yt, + ): + for channel in due: + if quota.backfill_reserve_reached(): + break + try: + data = yt.get_playlist_items(channel.uploads_playlist_id, None) + # silent-ok: counted, and the one summary line below names the count — the client + # already logged the call's own failure (`youtube.api.error`). + except YouTubeError: + failed += 1 + continue + rows = [ + s + for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", [])) + if s is not None + ] + try: + new += insert_stubs(db, rows) + channel.feed_fallback_at = datetime.now(timezone.utc) + db.add(channel) + db.commit() + except Exception: # noqa: BLE001 — one channel must not stop the rest + db.rollback() + log.exception( + "Feed fallback could not store channel %s", + channel.id, + extra=logs.event("sync.rss.fallback_failed"), + ) + failed += 1 + continue + read += 1 + log.info( + "Feed fallback: read %d of %d channels through the API, %d new videos", + read, + len(due), + new, + extra=logs.event("sync.rss.fallback", read=read, due=len(due), new=new, failed=failed), + ) + return {"read": read, "new": new, "failed": failed} def run_enrich(db: Session, max_batches: int = 200, floor: int = 200) -> int: diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index 8b2f49b3..41f1b7bd 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -91,6 +91,7 @@ SPECS: tuple[ConfigSpec, ...] = ( ConfigSpec( "backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500 ), + ConfigSpec("rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168), # --- Shorts probe --- ConfigSpec( "shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600 diff --git a/backend/tests/test_feed_fallback.py b/backend/tests/test_feed_fallback.py new file mode 100644 index 00000000..2c5432dd --- /dev/null +++ b/backend/tests/test_feed_fallback.py @@ -0,0 +1,115 @@ +"""The RSS poll's API fallback (`sync/runner.run_feed_fallback`), DB-lane: which channels are read +through the Data API when their feed stays dark, and how often.""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from app import sysconfig +from app.models import Channel, Video +from app.sync import runner + +NOW = datetime.now(timezone.utc) +_PUB = datetime(2020, 1, 1, tzinfo=timezone.utc) + + +class _FakeYt: + def __init__(self, calls): + self.calls = calls + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def get_playlist_items(self, playlist_id, page_token=None): + self.calls.append(playlist_id) + cid = "UC" + playlist_id[2:] + return { + "items": [ + { + "contentDetails": {"videoId": f"{cid}-v1", "videoPublishedAt": NOW.isoformat()}, + "snippet": {"title": "new upload"}, + } + ] + } + + +@pytest.fixture +def calls(monkeypatch): + made: list[str] = [] + monkeypatch.setattr(runner, "get_service_user", lambda db: object()) + monkeypatch.setattr(runner, "YouTubeClient", lambda db, user: _FakeYt(made)) + monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: False) + return made + + +def _channel(db, cid, *, rss=None, fallback=None, uploads=True): + ch = Channel( + id=cid, + title=cid, + published_at=_PUB, + uploads_playlist_id=("UU" + cid[2:]) if uploads else None, + last_rss_at=rss, + feed_fallback_at=fallback, + ) + db.add(ch) + db.commit() + return ch + + +def test_a_feed_dark_past_the_window_is_read_through_the_api(db, calls): + _channel(db, "UCdark", rss=NOW - timedelta(hours=3)) + + out = runner.run_feed_fallback(db, ["UCdark"]) + + assert out == {"read": 1, "new": 1, "failed": 0} + assert calls == ["UUdark"] + assert db.get(Video, "UCdark-v1") is not None + assert db.get(Channel, "UCdark").feed_fallback_at is not None + + +@pytest.mark.parametrize( + ("rss", "fallback"), + [ + (NOW - timedelta(minutes=30), None), # the feed answered recently: a blip, not an outage + (NOW - timedelta(hours=5), NOW - timedelta(minutes=30)), # already read this window + ], +) +def test_a_channel_read_within_the_window_is_left_alone(db, calls, rss, fallback): + _channel(db, "UCrecent", rss=rss, fallback=fallback) + + assert runner.run_feed_fallback(db, ["UCrecent"])["read"] == 0 + assert calls == [] + + +def test_the_window_is_the_admins_and_zero_turns_it_off(db, calls): + _channel(db, "UCdark", rss=NOW - timedelta(hours=3)) + + sysconfig.set_value(db, "rss_fallback_hours", 4) + db.commit() + assert runner.run_feed_fallback(db, ["UCdark"])["read"] == 0 + + sysconfig.set_value(db, "rss_fallback_hours", 0) + db.commit() + _channel(db, "UCnever") + assert runner.run_feed_fallback(db, ["UCdark", "UCnever"])["read"] == 0 + assert calls == [] + + +def test_only_the_named_channels_and_only_those_with_an_uploads_playlist(db, calls): + _channel(db, "UCdark") + _channel(db, "UCother") + _channel(db, "UCnolist", uploads=False) + + assert runner.run_feed_fallback(db, ["UCdark", "UCnolist"])["read"] == 1 + assert calls == ["UUdark"] + + +def test_it_yields_to_the_backfill_reserve(db, calls, monkeypatch): + _channel(db, "UCdark") + monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: True) + + assert runner.run_feed_fallback(db, ["UCdark"])["read"] == 0 + assert calls == [] diff --git a/backend/tests/test_followed_channels.py b/backend/tests/test_followed_channels.py index 798b972f..5b914695 100644 --- a/backend/tests/test_followed_channels.py +++ b/backend/tests/test_followed_channels.py @@ -27,7 +27,7 @@ def test_the_rss_poll_skips_channels_nobody_follows(db, make_user, monkeypatch): polled: list[str] = [] monkeypatch.setattr(runner, "fetch_channel_feed", lambda cid: polled.append(cid) or []) monkeypatch.setattr(runner, "apply_rss_feed", lambda db, ch, entries: 0) - assert runner.run_rss_poll(db) == {"new": 0, "failed": 0} + assert runner.run_rss_poll(db) == {"new": 0, "failed": 0, "fallback": 0} assert sorted(polled) == ["UC_explored", "UC_sub"] diff --git a/backend/tests/test_sync_guards.py b/backend/tests/test_sync_guards.py index b36ce352..a43dbafd 100644 --- a/backend/tests/test_sync_guards.py +++ b/backend/tests/test_sync_guards.py @@ -112,11 +112,15 @@ class TestRssPollResult: def _run(self, monkeypatch, *, fetch, apply_, channels=("UC1", "UC2")): monkeypatch.setattr(runner, "fetch_channel_feed", fetch) monkeypatch.setattr(runner, "apply_rss_feed", apply_) + # The API fallback has its own DB-lane tests (test_feed_fallback.py); here it reads nothing. + monkeypatch.setattr( + runner, "run_feed_fallback", lambda db, ids: {"read": 0, "new": 0, "failed": 0} + ) return runner.run_rss_poll(_StubDb(), [_StubChannel(c) for c in channels]) def test_a_healthy_run_reports_no_failures(self, monkeypatch): out = self._run(monkeypatch, fetch=lambda cid: [{"id": "v"}], apply_=lambda db, ch, e: 1) - assert out == {"new": 2, "failed": 0} + assert out == {"new": 2, "failed": 0, "fallback": 0} def test_unreadable_feeds_are_counted(self, monkeypatch): # One dead channel among healthy ones: counted, but the run itself is still a run. @@ -128,7 +132,7 @@ class TestRssPollResult: out = self._run( monkeypatch, fetch=fetch, apply_=lambda db, ch, e: 1, channels=("UC1", "UC_dead") ) - assert out == {"new": 1, "failed": 1} + assert out == {"new": 1, "failed": 1, "fallback": 0} def test_apply_failures_are_counted_too(self, monkeypatch): # A per-row write failure (a constraint clash on one channel) while the rest store fine. @@ -143,7 +147,9 @@ class TestRssPollResult: apply_=apply_, channels=("UC1", "UC_bad"), ) - assert out == {"new": 1, "failed": 1}, "an unstored channel must not read as ok" + assert out == {"new": 1, "failed": 1, "fallback": 0}, ( + "an unstored channel must not read as ok" + ) def test_many_failures_write_one_warning_with_counts_not_one_per_channel( self, monkeypatch, caplog @@ -163,7 +169,7 @@ class TestRssPollResult: caplog.set_level(logging.DEBUG, logger="siftlode.sync") out = self._run(monkeypatch, fetch=fetch, apply_=lambda db, ch, e: 1, channels=channels) - assert out == {"new": 5, "failed": 35} + assert out == {"new": 5, "failed": 35, "fallback": 0} warnings = [r for r in caplog.records if r.levelno >= logging.WARNING] assert len(warnings) == 1 assert warnings[0].event == "sync.rss.failed" @@ -195,7 +201,11 @@ class TestRssPollResult: def test_a_quiet_hour_is_distinguishable_from_an_outage(self, monkeypatch): quiet = self._run(monkeypatch, fetch=lambda cid: [], apply_=lambda db, ch, e: 0) - assert quiet == {"new": 0, "failed": 0} # same "new", different "failed" — that's the point + assert quiet == { + "new": 0, + "failed": 0, + "fallback": 0, + } # same "new", different "failed" — that's the point def test_a_total_outage_raises_instead_of_returning_ok(self, monkeypatch): # Every feed unreachable (the tinyproxy-lost-the-boot-race incident). Returning a result @@ -225,10 +235,38 @@ class TestRssPollResult: with pytest.raises(RuntimeError, match="read-only transaction"): self._run(monkeypatch, fetch=lambda cid: [{"id": "v"}], apply_=boom) + def test_only_the_unreadable_feeds_go_to_the_api_and_what_it_reads_counts(self, monkeypatch): + # A feed that answered 404 is the fallback's; a channel whose rows failed to STORE is not — + # the API would hand the same rows to the same broken write. + seen = [] + + def fetch(cid): + if cid == "UC_dark": + raise rss.RssError("404", "http_404") + return [{"id": "v"}] + + def apply_(db, ch, entries): + if ch.id == "UC_badrow": + raise RuntimeError("constraint") + return 1 + + def fallback(db, ids): + seen.extend(ids) + return {"read": 1, "new": 3, "failed": 0} + + monkeypatch.setattr(runner, "fetch_channel_feed", fetch) + monkeypatch.setattr(runner, "apply_rss_feed", apply_) + monkeypatch.setattr(runner, "run_feed_fallback", fallback) + out = runner.run_rss_poll( + _StubDb(), [_StubChannel(c) for c in ("UC_ok", "UC_dark", "UC_badrow")] + ) + assert seen == ["UC_dark"] + assert out == {"new": 4, "failed": 2, "fallback": 1} + def test_no_channels_at_all_is_not_an_outage(self, monkeypatch): # A fresh instance with nothing subscribed must not report an error every poll. out = self._run(monkeypatch, fetch=lambda cid: [], apply_=lambda db, ch, e: 0, channels=()) - assert out == {"new": 0, "failed": 0} + assert out == {"new": 0, "failed": 0, "fallback": 0} class TestRssErrorVsEmpty: diff --git a/frontend/src/i18n/locales/en/config.json b/frontend/src/i18n/locales/en/config.json index abf6d62c..586f0393 100644 --- a/frontend/src/i18n/locales/en/config.json +++ b/frontend/src/i18n/locales/en/config.json @@ -63,6 +63,10 @@ "label": "Recent backfill — max age (days)", "hint": "How far back the first pass reaches per channel." }, + "rss_fallback_hours": { + "label": "Feed fallback after (hours)", + "hint": "When a channel's RSS feed has not answered for this many hours, its newest uploads are read through the API instead (1 unit), at most once per this many hours. 0 turns it off." + }, "shorts_probe_max_seconds": { "label": "Shorts probe — max duration (s)", "hint": "Only videos at or below this length are probed as possible Shorts." diff --git a/frontend/src/i18n/locales/en/quotaActions.json b/frontend/src/i18n/locales/en/quotaActions.json index 2d8d98e0..7bec3cd4 100644 --- a/frontend/src/i18n/locales/en/quotaActions.json +++ b/frontend/src/i18n/locales/en/quotaActions.json @@ -6,6 +6,7 @@ "playlists_delete": "Delete playlist", "videos_backfill_recent": "Recent backfill", "videos_backfill_full": "Full-history backfill", + "videos_feed_fallback": "Feed fallback (RSS down)", "videos_enrich": "Video enrichment", "videos_lookup": "Video lookup", "videos_search": "Live search", diff --git a/frontend/src/i18n/locales/hu/config.json b/frontend/src/i18n/locales/hu/config.json index c32c69b4..320d9f3c 100644 --- a/frontend/src/i18n/locales/hu/config.json +++ b/frontend/src/i18n/locales/hu/config.json @@ -63,6 +63,10 @@ "label": "Friss letöltés — max kor (nap)", "hint": "Az első kör csatornánként meddig nyúl vissza." }, + "rss_fallback_hours": { + "label": "Feed-pótlás ennyi óra után", + "hint": "Ha egy csatorna RSS-feedje ennyi órája nem válaszol, a legújabb feltöltéseit az API-n keresztül olvassuk be (1 egység), csatornánként legfeljebb ennyi óránként egyszer. 0 = kikapcsolva." + }, "shorts_probe_max_seconds": { "label": "Shorts-vizsgálat — max hossz (mp)", "hint": "Csak az ennél nem hosszabb videókat vizsgáljuk lehetséges Shortsként." diff --git a/frontend/src/i18n/locales/hu/quotaActions.json b/frontend/src/i18n/locales/hu/quotaActions.json index bd688665..fb567bce 100644 --- a/frontend/src/i18n/locales/hu/quotaActions.json +++ b/frontend/src/i18n/locales/hu/quotaActions.json @@ -6,6 +6,7 @@ "playlists_delete": "Lejátszási lista törlése", "videos_backfill_recent": "Friss feltöltések betöltése", "videos_backfill_full": "Teljes előzmény betöltése", + "videos_feed_fallback": "Feed-pótlás (RSS kiesés)", "videos_enrich": "Videó-gazdagítás", "videos_lookup": "Videó-lekérdezés", "videos_search": "Élő keresés", From 3ee00fb0510e91124fc52a2a3c9d7fb1e88f8016 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 19 Sep 2026 04:51:31 +0200 Subject: [PATCH 02/21] fix(logs): keep polling reads and a channel-less account out of the warnings The reads the frontend polls (unread counts, /api/me, the download index, sync status, subtitle progress) are logged at DEBUG: at INFO they were ~95% of the stored log on prod. A scheduler job failure and a playlist sync failure carry event codes, and a Google account without a YouTube channel has an empty playlist list instead of a warning every six hours. --- backend/app/logs.py | 25 +++++++++++++++- backend/app/scheduler.py | 2 +- backend/app/sync/playlists.py | 14 +++++++-- backend/app/youtube/client.py | 35 ++++++++++++++++++---- backend/tests/test_logs.py | 32 ++++++++++++++++++++ backend/tests/test_youtube_no_channel.py | 38 ++++++++++++++++++++++++ 6 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 backend/tests/test_youtube_no_channel.py diff --git a/backend/app/logs.py b/backend/app/logs.py index cfb49e7c..2dbb9979 100644 --- a/backend/app/logs.py +++ b/backend/app/logs.py @@ -644,6 +644,27 @@ _REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,64}$") # — this lowers the floor, it does not hide. _QUIET_PREFIXES = ("/healthz", "/assets/", "/static/", "/api/admin/logs") +# The reads the frontend POLLS (a `useLiveQuery`, the `me` heartbeat, the sync status, the subtitle +# progress), by route template, GET only — logged at DEBUG like the prefixes above. One open tab asked +# for the two unread counts, `/api/me` and the download index every 30 s, and at INFO those were ~95% +# of the stored log (measured on prod 2026-09-19): 11k lines a day per tab burying every line someone +# caused. The same route answered with a failure still logs at WARNING/ERROR. +_POLLED_ROUTES = frozenset( + { + "/api/me", + "/api/me/notifications/unread_count", + "/api/messages/unread_count", + "/api/downloads", + "/api/downloads/index", + "/api/downloads/usage", + "/api/sync/my-status", + "/api/admin/scheduler", + "/api/admin/downloads", + "/api/admin/downloads/storage", + "/api/plex/subtitle/{rating_key}/{sub_ord}/status", + } +) + # Not `siftlode.logs`: this is the replacement for uvicorn's ACCESS log, and `siftlode.http` is # what an operator pins a level on when they want request lines and nothing else. The module it # happens to live in is not what the name is about. @@ -689,7 +710,9 @@ async def log_requests(request, call_next): raise duration_ms = round((time.perf_counter() - started) * 1000, 1) status = response.status_code - quiet = request.url.path.startswith(_QUIET_PREFIXES) + quiet = request.url.path.startswith(_QUIET_PREFIXES) or ( + request.method == "GET" and _route_template(request) in _POLLED_ROUTES + ) if status >= 500: level = logging.ERROR elif status >= 400: diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index ec2e8309..c3e24d12 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -208,7 +208,7 @@ def _job(name: str, fn) -> None: _audit_run(db, actor_id, name, "ok", summary, result) except Exception as exc: db.rollback() - log.exception("job %s failed", name) + log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name)) err = str(exc) or exc.__class__.__name__ _record(name, status="error", last_finished=_now_iso(), last_error=err, progress=None) if actor_id is not None: diff --git a/backend/app/sync/playlists.py b/backend/app/sync/playlists.py index 056c7a9d..2a889234 100644 --- a/backend/app/sync/playlists.py +++ b/backend/app/sync/playlists.py @@ -670,11 +670,19 @@ def sync_all_playlists(db: Session) -> dict: try: with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL): total += sync_user_playlists(db, user).get("synced", 0) - except YouTubeError: + except YouTubeError as exc: db.rollback() - log.warning("Playlist sync failed for user %s", user.id) + log.warning( + "Playlist sync failed for user %s", + user.id, + extra=logs.event("sync.playlists.failed", user_id=user.id, reason=exc.reason), + ) except Exception: db.rollback() - log.exception("Playlist sync crashed for user %s", user.id) + log.exception( + "Playlist sync crashed for user %s", + user.id, + extra=logs.event("sync.playlists.crashed", user_id=user.id), + ) progress.report(i, len(users), "playlist_sync") return {"users": len(users), "playlists_synced": total} diff --git a/backend/app/youtube/client.py b/backend/app/youtube/client.py index 260aff31..2e827c16 100644 --- a/backend/app/youtube/client.py +++ b/backend/app/youtube/client.py @@ -6,6 +6,7 @@ so they don't depend on a specific user's token; subscriptions.list?mine=true al uses OAuth. """ +import logging from collections.abc import Iterator from datetime import datetime, timedelta, timezone @@ -50,7 +51,9 @@ def _error_reason(resp: httpx.Response) -> str: return reasons or "" -def _upstream_failure(method: str, path: str, resp: httpx.Response) -> YouTubeError: +def _upstream_failure( + method: str, path: str, resp: httpx.Response, expected: frozenset[str] = frozenset() +) -> YouTubeError: """Report one upstream failure and build the error to raise. Every YouTube response that is not a success goes through here — the Data API's reads, writes and deletes and the OAuth token refresh — so what may appear in a log line cannot drift apart between them, and the @@ -59,7 +62,11 @@ def _upstream_failure(method: str, path: str, resp: httpx.Response) -> YouTubeEr response at all is `_send`'s, with the reason `transport`.) The body is logged only under the requesting user's own Teljes consent: it is the request quoted back.""" reason = _error_reason(resp) - log.warning( + # A reason the caller named as an answer rather than a failure (`channelNotFound` on a `mine` + # read: the account simply has no channel) is still raised, but it is not a warning — it came + # back every six hours for an account that will never have one. + log.log( + logging.DEBUG if reason in expected else logging.WARNING, "YouTube %s %s -> %s (%s)", method, path, @@ -83,6 +90,10 @@ def _upstream_failure(method: str, path: str, resp: httpx.Response) -> YouTubeEr return YouTubeError(f"{method} {path} -> {resp.status_code}: {reason}", reason=reason) +# The `mine` reads' answer for a Google account that has no YouTube channel. +_NO_CHANNEL = frozenset({"channelNotFound"}) + + def _chunks(items: list, size: int) -> Iterator[list]: for i in range(0, len(items), size): yield items[i : i + size] @@ -167,7 +178,14 @@ class YouTubeClient: return access # --- core request --- - def _get(self, path: str, params: dict, cost: int = 1, allow_key: bool = True) -> dict: + def _get( + self, + path: str, + params: dict, + cost: int = 1, + allow_key: bool = True, + expected: frozenset[str] = frozenset(), + ) -> dict: p = dict(params) headers = {} api_key = sysconfig.get_str(self.db, "youtube_api_key") @@ -178,7 +196,7 @@ class YouTubeClient: resp = self._send("GET", f"{API_BASE}/{path}", params=p, headers=headers) quota.record_usage(cost) if resp.status_code != 200: - raise _upstream_failure("GET", path, resp) + raise _upstream_failure("GET", path, resp, expected) return resp.json() # --- endpoints --- @@ -218,7 +236,14 @@ class YouTubeClient: params = {"part": "snippet,contentDetails", "mine": "true", "maxResults": 50} if page_token: params["pageToken"] = page_token - data = self._get("playlists", params, cost=1, allow_key=False) + try: + data = self._get("playlists", params, cost=1, allow_key=False, expected=_NO_CHANNEL) + except YouTubeError as exc: + # A Google account without a YouTube channel has no playlists, and YouTube says + # so as a 404. That is an empty list, not a failed sync. + if exc.reason in _NO_CHANNEL: + return + raise for item in data.get("items", []): sn = item.get("snippet", {}) yield { diff --git a/backend/tests/test_logs.py b/backend/tests/test_logs.py index a01403fb..ece53423 100644 --- a/backend/tests/test_logs.py +++ b/backend/tests/test_logs.py @@ -219,6 +219,14 @@ def _client() -> TestClient: raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) return {"ok": True} + @app.get("/api/messages/unread_count") + def _unread(): + return {"count": 0} + + @app.post("/api/messages/unread_count") + def _unread_write(): + return {"ok": True} + @app.get("/boom") def _boom(): raise RuntimeError("kaboom") @@ -323,6 +331,30 @@ def test_the_log_consoles_own_reads_are_debug_but_its_failures_are_not(sink): assert sink.objects()[-1]["level"] == "WARNING" +def test_a_polled_read_is_debug_a_write_to_the_same_route_is_not(sink): + # One open tab polled the unread counts, /api/me and the download index every 30 s: at INFO + # that was ~95% of the stored log on prod. + http = logging.getLogger("siftlode.http") + http.setLevel(logs.TRACE) + client = _client() + client.get("/api/messages/unread_count") + assert sink.objects()[-1]["level"] == "DEBUG" + client.post("/api/messages/unread_count") + assert sink.objects()[-1]["level"] == "INFO" + client.get("/api/videos/abc") + assert sink.objects()[-1]["level"] == "INFO" + + +def test_every_polled_route_is_a_route_the_app_has(): + # The list names route TEMPLATES; one renamed on the router side would silently fall back to + # INFO, so the list is held to the app's own routes. + from app.main import app + + # The schema, not `app.routes`: FastAPI 0.141 keeps an included router behind a wrapper there. + gets = {path for path, ops in app.openapi()["paths"].items() if "get" in ops} + assert logs._POLLED_ROUTES - gets == set() + + def test_a_failing_request_is_logged_by_status_and_an_unhandled_error_is_logged_at_all(sink): client = _client() client.get("/api/videos/missing/deeper") # 404 from the router diff --git a/backend/tests/test_youtube_no_channel.py b/backend/tests/test_youtube_no_channel.py new file mode 100644 index 00000000..a3c3d10f --- /dev/null +++ b/backend/tests/test_youtube_no_channel.py @@ -0,0 +1,38 @@ +"""A Google account without a YouTube channel: its `mine` playlists read answers 404 +`channelNotFound`. That is an empty list, not a failed sync — it came back as a warning every six +hours on prod for an account that will never have a channel.""" + +import logging + +import httpx +import pytest + +from app.youtube import client as yt + + +def _client(monkeypatch, status: int, reason: str) -> yt.YouTubeClient: + resp = httpx.Response( + status, + json={"error": {"code": status, "errors": [{"reason": reason, "message": "prose"}]}}, + request=httpx.Request("GET", f"{yt.API_BASE}/playlists"), + ) + c = yt.YouTubeClient.__new__(yt.YouTubeClient) # no HTTP client: `_send` is the stub + c.db = None + monkeypatch.setattr(yt.sysconfig, "get_str", lambda db, key: "") + monkeypatch.setattr(yt.quota, "record_usage", lambda cost: None) + monkeypatch.setattr(c, "_access_token", lambda: "t") + monkeypatch.setattr(c, "_send", lambda *a, **k: resp) + return c + + +def test_no_channel_is_an_empty_playlist_list_and_no_warning(monkeypatch, caplog): + caplog.set_level(logging.DEBUG, logger="siftlode") + assert list(_client(monkeypatch, 404, "channelNotFound").iter_my_playlists()) == [] + assert not [r for r in caplog.records if r.levelno >= logging.WARNING] + + +def test_any_other_failure_still_raises_and_warns(monkeypatch, caplog): + caplog.set_level(logging.DEBUG, logger="siftlode") + with pytest.raises(yt.YouTubeError): + list(_client(monkeypatch, 403, "quotaExceeded").iter_my_playlists()) + assert [r for r in caplog.records if r.levelno == logging.WARNING] From 51e98bfc3854cacf13aee41d46c189741c7f0e36 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 19 Sep 2026 04:51:31 +0200 Subject: [PATCH 03/21] fix(app): reload onto a new build instead of crashing on a missing chunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tab left open across a release asked for a lazy chunk the server no longer has (prod: DownloadDialog 404 an hour after 0.89) and fell to the crash screen. `vite:preloadError` now reloads the tab once a minute at most; a second failure is let through to the error boundary, which now wraps every entry — the public /watch, /privacy and /terms pages crashed to a blank page. --- frontend/src/components/ErrorBoundary.tsx | 5 ++ frontend/src/lib/staleBuild.dom.test.ts | 44 ++++++++++++ frontend/src/lib/staleBuild.ts | 54 ++++++++++++++ frontend/src/main.tsx | 85 ++++++++++++----------- 4 files changed, 148 insertions(+), 40 deletions(-) create mode 100644 frontend/src/lib/staleBuild.dom.test.ts create mode 100644 frontend/src/lib/staleBuild.ts diff --git a/frontend/src/components/ErrorBoundary.tsx b/frontend/src/components/ErrorBoundary.tsx index c30a288e..acfdb5f9 100644 --- a/frontend/src/components/ErrorBoundary.tsx +++ b/frontend/src/components/ErrorBoundary.tsx @@ -2,6 +2,7 @@ import { Component, type ErrorInfo, type ReactNode } from "react"; import i18n from "../i18n"; import { notify } from "../lib/notifications"; import { errorAttrs, log } from "../lib/log"; +import { reloadingForNewBuild } from "../lib/staleBuild"; interface Props { children: ReactNode; @@ -20,6 +21,9 @@ export default class ErrorBoundary extends Component { } componentDidCatch(error: Error, info: ErrorInfo) { + // The tab is already reloading onto a newer build (lib/staleBuild): this is the chunk it could not + // load, not a crash — no fatal notice, which would outlive the reload. + if (reloadingForNewBuild()) return; log.fatal("ui.crash", "the app crashed while rendering", { ...errorAttrs(error), ...(info.componentStack ? { component_stack: info.componentStack } : {}), @@ -34,6 +38,7 @@ export default class ErrorBoundary extends Component { render() { if (this.state.crashed) { + if (reloadingForNewBuild()) return null; return (
diff --git a/frontend/src/lib/staleBuild.dom.test.ts b/frontend/src/lib/staleBuild.dom.test.ts new file mode 100644 index 00000000..05ddeca7 --- /dev/null +++ b/frontend/src/lib/staleBuild.dom.test.ts @@ -0,0 +1,44 @@ +// @vitest-environment jsdom +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const fire = (): Event => { + const e = new Event("vite:preloadError", { cancelable: true }); + window.dispatchEvent(e); + return e; +}; + +let uninstall = () => {}; + +describe("a lazy chunk the server no longer has", () => { + afterEach(() => uninstall()); + beforeEach(() => { + vi.resetModules(); + sessionStorage.clear(); + }); + + it("reloads the tab once, and says so to the error boundary", async () => { + const { installStaleBuildReload, reloadingForNewBuild } = await import("./staleBuild"); + const reload = vi.fn(); + uninstall = installStaleBuildReload(reload); + + const e = fire(); + + expect(e.defaultPrevented).toBe(true); + expect(reload).toHaveBeenCalledTimes(1); + expect(reloadingForNewBuild()).toBe(true); + // The rest of the page's failed imports while the reload is under way: swallowed, no second reload. + expect(fire().defaultPrevented).toBe(true); + expect(reload).toHaveBeenCalledTimes(1); + }); + + it("lets the error through when the reload a moment ago did not help", async () => { + sessionStorage.setItem("sf.staleBuildReload", String(Date.now() - 5_000)); + const { installStaleBuildReload, reloadingForNewBuild } = await import("./staleBuild"); + const reload = vi.fn(); + uninstall = installStaleBuildReload(reload); + + expect(fire().defaultPrevented).toBe(false); + expect(reload).not.toHaveBeenCalled(); + expect(reloadingForNewBuild()).toBe(false); + }); +}); diff --git a/frontend/src/lib/staleBuild.ts b/frontend/src/lib/staleBuild.ts new file mode 100644 index 00000000..5a894028 --- /dev/null +++ b/frontend/src/lib/staleBuild.ts @@ -0,0 +1,54 @@ +// A tab left open across a release still runs the old build, and the old build's lazy chunks are gone +// from the server the moment the new one is deployed: its first lazy import (the download dialog, a +// module it had not opened yet) answers 404, and the whole app fell to the crash screen. Measured on +// prod 2026-09-19 — `GET /assets/DownloadDialog-….js 404` at 00:59, an hour after 0.89 went out. +// +// Vite announces such a failure as `vite:preloadError` (for the import itself as well as for its +// preloaded dependencies); cancelling it and reloading puts the tab on the current build. Once per +// minute at most: a chunk that is missing from the NEW build too would otherwise reload forever, so +// the second failure inside the minute is let through to the error boundary, which says so. +import { log } from "./log"; + +const KEY = "sf.staleBuildReload"; +const WINDOW_MS = 60_000; + +let reloading = false; + +/** True once a reload for a newer build is under way: whatever fails to render in the meantime is + * that missing chunk, and the error boundary must not report it as a crash. */ +export function reloadingForNewBuild(): boolean { + return reloading; +} + +function lastReload(): number { + try { + return Number(sessionStorage.getItem(KEY) ?? 0) || 0; + } catch { + return 0; + } +} + +/** Returns the uninstall (for tests: the listener is on `window`, which outlives a module reset). */ +export function installStaleBuildReload( + reload: () => void = () => window.location.reload(), +): () => void { + const onError = (event: Event) => { + if (reloading) { + event.preventDefault(); + return; + } + if (Date.now() - lastReload() < WINDOW_MS) return; + try { + sessionStorage.setItem(KEY, String(Date.now())); + } catch { + // No storage (a private window): without the marker a reload could repeat, so don't. + return; + } + event.preventDefault(); + reloading = true; + log.warn("app.stale_build", "a chunk of this build is gone from the server; reloading"); + reload(); + }; + window.addEventListener("vite:preloadError", onError); + return () => window.removeEventListener("vite:preloadError", onError); +} diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index bd4f567f..2b50c27d 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -21,10 +21,13 @@ import "./index.css"; import "flag-icons/css/flag-icons.min.css"; import { lazyWithPreload } from "./lib/lazyWithPreload"; import { installClientLog } from "./lib/logHooks"; +import { installStaleBuildReload } from "./lib/staleBuild"; import { appCopyOf } from "./i18n/appCopy"; // First, before any render: a crash or a navigation during boot belongs in the log too. installClientLog(appCopyOf); +// A tab from before the last release reloads onto the new build instead of crashing on a lazy chunk. +installStaleBuildReload(); // Split by top-level route so each entry point is its own chunk: a public /watch share link or a // legal page never downloads the authenticated app bundle, and the app never carries them either. @@ -63,50 +66,52 @@ const root = ) : ( - - {/* One tree for the whole app: every anchored popover registers as a node so Escape and - outside-press act only on the topmost layer (S3 folds the Modal + players in). */} - - - {/* Nothing below renders until the tab knows which account it is: every provider - seeds per-account state in its initializer (see BootGate). */} - - - - - - - {/* SavedViewsProvider sits below BOTH filter providers so it can switch its - source (feed ⟷ plex) by the active page and read the right live filters. */} - - - - - - {/* The same quiet screen as the gate's, so a chunk that is still - loading when the gate opens continues it instead of blanking. */} - }> - - - - - - - - - - - - - - - - + {/* One tree for the whole app: every anchored popover registers as a node so Escape and + outside-press act only on the topmost layer (S3 folds the Modal + players in). */} + + + {/* Nothing below renders until the tab knows which account it is: every provider + seeds per-account state in its initializer (see BootGate). */} + + + + + + + {/* SavedViewsProvider sits below BOTH filter providers so it can switch its + source (feed ⟷ plex) by the active page and read the right live filters. */} + + + + + + {/* The same quiet screen as the gate's, so a chunk that is still + loading when the gate opens continues it instead of blanking. */} + }> + + + + + + + + + + + + + + + ); createRoot(document.getElementById("root")!).render( - }>{root} + {/* Around EVERY entry, the public ones too: a crash on a shared /watch link, or its chunk gone + twice over after a release (lib/staleBuild), left a blank white page with no way on. */} + + }>{root} + , ); From 6096dd69273cbe0d76972f3610ce2632daf9191b Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 19 Sep 2026 04:51:42 +0200 Subject: [PATCH 04/21] fix(diagnostics): Back from a request's lines returns to the view it came from Every filter change replaced the history entry, so Back after a request or job pivot left the console altogether (the 0.90 notes promised otherwise). A pivot follows a link and is its own entry now; the console reads the landed entry back on popstate. The other filters still adjust in place. --- .../src/components/AdminLogs.dom.test.tsx | 36 +++++++++++++++++++ frontend/src/components/AdminLogs.tsx | 24 +++++++++++-- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/AdminLogs.dom.test.tsx b/frontend/src/components/AdminLogs.dom.test.tsx index 8e6891cd..eba9bec0 100644 --- a/frontend/src/components/AdminLogs.dom.test.tsx +++ b/frontend/src/components/AdminLogs.dom.test.tsx @@ -124,6 +124,42 @@ describe("AdminLogs", () => { expect([url.get("lreq"), url.get("lgrp"), url.get("lq")]).toEqual(["req-00000001", null, null]); }); + it("Back from a request's lines returns to the view the pivot was made from", async () => { + window.history.replaceState({ sfPage: "logs" }, "", "/?page=logs&lgrp=error&lq=boom"); + const depth = window.history.length; + logsMock.mockResolvedValue({ items: [row(1)], accounts: {}, next_cursor: null }); + renderConsole(); + await waitFor(() => expect(lines()).toHaveLength(1)); + fireEvent.click(lines()[0]!); + fireEvent.click(screen.getByRole("button", { name: "req-00000001" })); + await waitFor(() => + expect(screen.getByText("adminLogs.pivotRequest:req-00000001")).toBeTruthy(), + ); + expect(window.history.length).toBe(depth + 1); + + await act(async () => { + const landed = new Promise((resolve) => + window.addEventListener("popstate", resolve, { once: true }), + ); + window.history.back(); + await landed; + }); + + await waitFor(() => + expect(logsMock).toHaveBeenLastCalledWith( + expect.objectContaining({ + request_id: undefined, + levels: ["ERROR", "CRITICAL"], + q: "boom", + }), + null, + ), + ); + expect(screen.queryByText("adminLogs.pivotRequest:req-00000001")).toBeNull(); + const url = new URLSearchParams(window.location.search); + expect([url.get("lreq"), url.get("lgrp"), url.get("lq")]).toEqual([null, "error", "boom"]); + }); + it("keeps the account in use offered when the facets no longer count it", async () => { window.history.replaceState(null, "", "/?page=logs&lu=7"); logsMock.mockResolvedValue({ items: [], accounts: {}, next_cursor: null }); diff --git a/frontend/src/components/AdminLogs.tsx b/frontend/src/components/AdminLogs.tsx index 1f5754db..9ecd017e 100644 --- a/frontend/src/components/AdminLogs.tsx +++ b/frontend/src/components/AdminLogs.tsx @@ -1,4 +1,4 @@ -import { useMemo, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { keepPreviousData, @@ -11,6 +11,7 @@ import { Copy, Radio, Search, X } from "lucide-react"; import { api, type LogFacet, type LogQuery, type ServerLogRow } from "../lib/api"; import { copyText } from "../lib/clipboard"; import { formatDateTime } from "../lib/format"; +import { nextNavId, notifyNav, rememberNavPosition } from "../lib/history"; import { LEVEL_GROUPS, type LevelGroup } from "../lib/log"; import { DEFAULT_LOGS_STATE, @@ -23,6 +24,7 @@ import { import { notify } from "../lib/notifications"; import { qk } from "../lib/queryKeys"; import { useDebounced } from "../lib/useDebounced"; +import { urlWithParams } from "../lib/urlState"; import { useUrlMirror } from "../lib/useUrlMirror"; import LogLine from "./LogLine"; import NavGrid from "./ui/NavGrid"; @@ -82,6 +84,15 @@ export default function AdminLogs() { const [open, setOpen] = useState(null); const q = useDebounced(state.q, 300); useUrlMirror(logsParamUpdates(state)); + // Back/Forward lands on another view of the console (a pivot pushed one, below): read it back. + useEffect(() => { + const onPop = () => { + setOpen(null); + setState(logsParamsToState(new URLSearchParams(window.location.search))); + }; + window.addEventListener("popstate", onPop); + return () => window.removeEventListener("popstate", onPop); + }, []); const update = (patch: Partial) => { setOpen(null); @@ -184,9 +195,18 @@ export default function AdminLogs() { return facets.data ? sum(levels.filter((f) => inGroup.includes(f.value))) : undefined; }; + // A pivot follows a link to another view, so it is its own history entry and Back returns to the + // view it was made from. The other filters are adjustments of the view on screen and replace it. const pivot = (patch: Pick) => { setLive(false); - update({ ...PIVOT_RESET, ...patch }); + const next = { ...state, ...PIVOT_RESET, ...patch }; + window.history.pushState( + { ...rememberNavPosition(), sfScroll: nextNavId() }, + "", + urlWithParams(logsParamUpdates(next)), + ); + notifyNav(); + update(next); }; // "Any", then what the facet counted. The value in use stays offered even at zero — under the From 250e3a9f90ce1253be3fdf059d81858a5db71111 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 19 Sep 2026 04:51:43 +0200 Subject: [PATCH 05/21] fix(home): Back keeps the question, and the focus finds its card by name Opening a card left Home, and Back came back to "Doesn't matter" with a different hero, whose card then dropped out of the Continue rail and moved the keyboard onto its neighbour. The budget, reroll seed and seen list live on the history entry now, and a returning list focuses the item by its name before its place. --- frontend/src/components/Home.dom.test.tsx | 26 +++++++++++ frontend/src/components/Home.pin.dom.test.tsx | 2 + frontend/src/components/Home.tsx | 40 +++++++++++++++-- .../src/lib/history.navPosition.dom.test.tsx | 24 +++++++++++ frontend/src/lib/listNav.ts | 43 +++++++++++++------ 5 files changed, 119 insertions(+), 16 deletions(-) diff --git a/frontend/src/components/Home.dom.test.tsx b/frontend/src/components/Home.dom.test.tsx index 50017494..4c2aab42 100644 --- a/frontend/src/components/Home.dom.test.tsx +++ b/frontend/src/components/Home.dom.test.tsx @@ -163,6 +163,8 @@ function mountWithClient() { } beforeEach(() => { + // Home keeps its question on the history entry (Back restores it): each test starts a fresh one. + window.history.replaceState(null, ""); discover.mockReset(); discoverPick.mockReset(); setPage.mockReset(); @@ -553,11 +555,35 @@ describe("Home", () => { first.unmount(); discoverPick.mockClear(); + // A later visit is a new history entry (setPage pushes a fresh one); the same entry would bring + // the rerolled question back on purpose (see the Back test below). + window.history.pushState(null, ""); mount(); await screen.findByText("Hero v1"); expect(discoverPick).not.toHaveBeenCalled(); }); + // Prod UAT 2026-09-19: a card opened from Home leaves the page, and Back came back to "Doesn't + // matter" and a different hero. The question is kept on the entry, so Back asks it again. + it("Back to the same entry brings back the budget and the rerolled hero", async () => { + discoverPick.mockResolvedValue({ + budget: "short", + hero: { tier: "surprise", card: ytCard("v9", "Rerolled") }, + }); + const first = mount(); + await screen.findByText("Hero v1"); + fireEvent.click(screen.getByText("home.budget.short")); + fireEvent.click(screen.getByText("home.hero.reroll")); + expect(await screen.findByText("Rerolled")).toBeTruthy(); + first.unmount(); + + discover.mockClear(); + mount(); + + expect(await screen.findByText("Rerolled")).toBeTruthy(); + expect(discover).toHaveBeenLastCalledWith("short", 0); + }); + // Measured in the browser (2026-09-07), before the fix: the palette's "Meglepetés" arrived at Home // as `start=2` — two picks deep before the page had painted once — because StrictMode invokes a // mount effect twice with the same render snapshot, and a reroll is not idempotent. The token is diff --git a/frontend/src/components/Home.pin.dom.test.tsx b/frontend/src/components/Home.pin.dom.test.tsx index 9b122160..6489ab53 100644 --- a/frontend/src/components/Home.pin.dom.test.tsx +++ b/frontend/src/components/Home.pin.dom.test.tsx @@ -149,6 +149,8 @@ const item = (label: string) => { }; beforeEach(() => { + // Home keeps its question on the history entry (Back restores it): each test starts a fresh one. + window.history.replaceState(null, ""); discover.mockReset().mockResolvedValue(payload()); comfortPins.mockReset().mockResolvedValue({ items: shelf }); addComfortPin.mockReset().mockResolvedValue({ ok: true, id: 1, created: true }); diff --git a/frontend/src/components/Home.tsx b/frontend/src/components/Home.tsx index 9ab7b13b..454f5098 100644 --- a/frontend/src/components/Home.tsx +++ b/frontend/src/components/Home.tsx @@ -348,26 +348,58 @@ function Hero({ ); } +const SAVED_KEY = "sfHome"; + +interface SavedQuestion { + budget: DiscoverBudget; + seed: number; + press: number; + seen: string[]; +} + +/** The question this history entry was left with, if it was left from Home. Read from + * `history.state`, which only this app writes — still checked field by field. */ +function readSavedQuestion(): SavedQuestion | null { + const raw = (window.history.state as Record | null)?.[SAVED_KEY] as + | Partial + | undefined; + if (!raw || !DISCOVER_BUDGETS.includes(raw.budget as DiscoverBudget)) return null; + return { + budget: raw.budget as DiscoverBudget, + seed: Number.isInteger(raw.seed) ? (raw.seed as number) : 0, + press: Number.isInteger(raw.press) ? (raw.press as number) : 0, + seen: Array.isArray(raw.seen) ? raw.seen.filter((k) => typeof k === "string") : [], + }; +} + export default function Home() { const { t } = useTranslation(); const qc = useQueryClient(); const { setPage } = useNavigationActions(); const { setJumpTo } = usePlexActions(); - const [budget, setBudget] = useState("any"); + // The question on screen lives on this page's history ENTRY as well: opening a card leaves Home + // (it unmounts), and Back used to come back to "Doesn't matter" and a different hero — whose card + // then left the Continue rail, so the keyboard landed on its neighbour (prod UAT 2026-09-19). + const [saved] = useState(readSavedQuestion); + const [budget, setBudget] = useState(saved?.budget ?? "any"); // 0 = "the hero the page loaded with". Any other value is a reroll, answered by its own query so // pressing the button does not refetch the rails under the user's cursor. - const [seed, setSeed] = useState(0); + const [seed, setSeed] = useState(saved?.seed ?? 0); // Which press this is. It rotates the tier the server leads with, so "Mást kérek" offers a // different KIND of answer each time (where you left off → your shelf → a surprise) rather than // the next row of the same rail. Reset with the budget: a new question starts the rotation over. - const [press, setPress] = useState(0); + const [press, setPress] = useState(saved?.press ?? 0); // The cards the reroll has already offered this session, newest last. A ROLLING window rather than // the whole history: an unbounded list would eventually exclude everything and leave the button // with nothing to give, whereas a bounded one cannot. Its size is `SEEN_WINDOW` — see there for // why it is 32 and not the 8 this comment used to quote. Kept here (not in the URL or a pref) // because it is a property of this sitting. - const [seen, setSeen] = useState([]); + const [seen, setSeen] = useState(saved?.seen ?? []); + useEffect(() => { + const question: SavedQuestion = { budget, seed, press, seen }; + window.history.replaceState({ ...window.history.state, [SAVED_KEY]: question }, ""); + }, [budget, seed, press, seen]); const [player, setPlayer] = useState<{ video: Video; startAt: number | null } | null>(null); const q = useQuery({ diff --git a/frontend/src/lib/history.navPosition.dom.test.tsx b/frontend/src/lib/history.navPosition.dom.test.tsx index 9ee1da77..b9c080a4 100644 --- a/frontend/src/lib/history.navPosition.dom.test.tsx +++ b/frontend/src/lib/history.navPosition.dom.test.tsx @@ -170,6 +170,30 @@ describe("rememberNavPosition · the list it comes back to (deep-review E11 roun await waitFor(() => expect(focused()).toBe("a1")); }); + // Home's Continue rail leaves out whatever the hero shows: back from a card, a different hero put a + // card back in front of it, and by its place the focus landed on the neighbour (prod UAT 2026-09-19). + it("finds the item by its name when the rail's contents moved", async () => { + const Rail = ({ titles }: { titles: string[] }) => { + const ref = useRef(null); + useListNav(ref, { enabled: true }); + return ( +
+ {titles.map((title, i) => ( +
+ {title} +
+ ))} +
+ ); + }; + const { unmount } = render(); + const stamped = leaveFrom("I Am Mother", 2); + unmount(); + landOn(stamped); + render(); + await waitFor(() => expect(focused()).toBe("I Am Mother")); + }); + it("gives up on an unnamed list when the page came back with another number of lists", async () => { const { unmount } = render(); const stamped = leaveFrom("a1", 2); diff --git a/frontend/src/lib/listNav.ts b/frontend/src/lib/listNav.ts index 330fe48e..4b143d64 100644 --- a/frontend/src/lib/listNav.ts +++ b/frontend/src/lib/listNav.ts @@ -164,6 +164,18 @@ function slotOfRemoved(delta: 1 | -1): { list: NavList; index: number } | null { : null; } +/** What an item is called: its own name when it gives one (`navItemAttrs`), else the open element's + * label or text — a title link's text is the title (deep-review E11 round 3). */ +function itemName(el: HTMLElement): string { + const open = el.querySelector("[data-nav-open]"); + return ( + el.getAttribute("data-nav-name") || + open?.getAttribute("aria-label") || + open?.textContent || + "" + ).trim(); +} + function focusItem(el: HTMLElement, reveal = true): void { if (roved !== el) { if (named) roved?.removeAttribute("role"); @@ -179,15 +191,7 @@ function focusItem(el: HTMLElement, reveal = true): void { // A card root is a plain
: focused, it has no role and no name, and a screen reader read it // as an unlabelled run of text. While it holds the focus it is a group named after what Enter // opens (deep-review E11 round 2). A row () keeps its own role. - // The item's own name when it gives one (`navItemAttrs`), else the open element's label or text: - // a title link's text is the title (deep-review E11 round 3). - const open = el.querySelector("[data-nav-open]"); - const name = ( - el.getAttribute("data-nav-name") || - open?.getAttribute("aria-label") || - open?.textContent || - "" - ).trim(); + const name = itemName(el); if ( el.tagName === "DIV" && name && @@ -321,12 +325,14 @@ function onKey(e: KeyboardEvent): void { // on screen again. An overlay (the YouTube player) never unmounts the list; it restores focus itself. /** Where focus sits in the page's lists: the list's place in reading order, how many lists the page - * had on screen, the list's accessible name when it has one (a rail's title), and the item's index. */ + * had on screen, the list's accessible name when it has one (a rail's title), the item's index, and + * the item's own name. */ export interface NavPosition { readonly list: number; readonly lists: number; readonly label?: string | null; readonly index: number; + readonly name?: string | null; } const labelOf = (list: NavList): string | null => list.el.getAttribute("aria-label"); @@ -336,7 +342,15 @@ export function navPosition(): NavPosition | null { if (!cur) return null; const all = ordered(); const list = all.indexOf(cur.list); - return list < 0 ? null : { list, lists: all.length, label: labelOf(cur.list), index: cur.index }; + return list < 0 + ? null + : { + list, + lists: all.length, + label: labelOf(cur.list), + index: cur.index, + name: itemName(cur.item) || null, + }; } // How long a returning page gets to put its lists back (a lazy chunk, cached data), and how often we @@ -364,7 +378,12 @@ export function restoreNavPosition(pos: NavPosition): () => void { : all.length === pos.lists ? all[pos.list] : undefined; - const el = list ? itemAt(list, pos.index) : null; + // The item by its NAME first, its place second: a rail's contents can shift while the page was + // away — Home's Continue rail leaves out whatever the hero shows, so a different hero moved every + // card after it one place and the focus landed on the neighbour (prod UAT 2026-09-19). + const named = + list && pos.name ? ownItems(list.el).find((it) => itemName(it) === pos.name) : null; + const el = named ?? (list ? itemAt(list, pos.index) : null); if (el) return focusItem(el, false); if (Date.now() < deadline) timer = setTimeout(attempt, RESTORE_POLL_MS); }; From ef652ee5d8fbdf98a4edeeaf11145671c2a15a22 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 19 Sep 2026 04:51:43 +0200 Subject: [PATCH 06/21] fix(ui): honest tab titles and a download field that names every site Signed out, the tab read Home's question over the sign-in form; the legal pages read just "Siftlode". The download field asked only for a YouTube link although Facebook and the rest download too. --- .../NavigationProvider.dom.test.tsx | 27 ++++++++++++++++++- .../src/components/NavigationProvider.tsx | 19 ++++++++----- frontend/src/components/legal/LegalLayout.tsx | 5 ++++ frontend/src/i18n/locales/en/downloads.json | 2 +- frontend/src/i18n/locales/hu/downloads.json | 2 +- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/frontend/src/components/NavigationProvider.dom.test.tsx b/frontend/src/components/NavigationProvider.dom.test.tsx index 0d19f2ce..fe29cf5d 100644 --- a/frontend/src/components/NavigationProvider.dom.test.tsx +++ b/frontend/src/components/NavigationProvider.dom.test.tsx @@ -11,6 +11,7 @@ // bookkeeping. import { describe, it, expect, afterEach, beforeEach, vi } from "vitest"; import { renderHook, act, cleanup } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import type { ReactNode } from "react"; // Partial: the provider reads `t` AND `i18n.language` (both feed the document-title effect), and the @@ -28,8 +29,14 @@ import { NavigationProvider } from "./NavigationProvider"; import { useNavigation, useNavigationActions } from "./navigationContext"; import { useBackToClose } from "../lib/history"; +import { qk } from "../lib/queryKeys"; + +// The provider reads the signed-in account (the tab title) from the `me` cache. +let qc = new QueryClient(); const wrapper = ({ children }: { children: ReactNode }) => ( - {children} + + {children} + ); const render = () => renderHook(() => ({ ...useNavigation(), ...useNavigationActions() }), { wrapper }); @@ -54,6 +61,8 @@ async function openOverlay() { } beforeEach(() => { + qc = new QueryClient(); + qc.setQueryData(qk.me(), { id: 1 }); localStorage.clear(); sessionStorage.setItem("siftlode.activeAccount", "1"); // account-scoped page memory window.history.replaceState({ sfPage: "feed" }, "", "/"); @@ -65,6 +74,22 @@ afterEach(() => { vi.restoreAllMocks(); }); +describe("NavigationProvider · the tab title", () => { + // Prod UAT 2026-09-19: signed out, the tab read the Home page's question over the sign-in form. + it("is the brand alone while nobody is signed in, the page's name once someone is", async () => { + window.history.replaceState({ sfPage: "home" }, "", "/?page=home"); + qc.setQueryData(qk.me(), null); + render(); + await settle(); + expect(document.title).toBe("Siftlode"); + + act(() => qc.setQueryData(qk.me(), { id: 1 })); + await settle(); + expect(document.title).not.toBe("Siftlode"); + expect(document.title.endsWith(" · Siftlode")).toBe(true); + }); +}); + describe("NavigationProvider · setPage's same-page guard", () => { it("pushes an entry for a real page change", () => { const { result } = render(); diff --git a/frontend/src/components/NavigationProvider.tsx b/frontend/src/components/NavigationProvider.tsx index 82faf9a8..b196bcce 100644 --- a/frontend/src/components/NavigationProvider.tsx +++ b/frontend/src/components/NavigationProvider.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { useTranslation } from "react-i18next"; +import { useMe } from "../lib/useMe"; import { urlWithParams, type Page } from "../lib/urlState"; import { LOGS_KEYS } from "../lib/logsUrlState"; import { pageTitleKey } from "../lib/pageMeta"; @@ -154,16 +155,20 @@ export function NavigationProvider({ children }: { children: ReactNode }) { }, [page]); // Reflect the current module in the browser tab title. Feed = brand only; an open channel page - // shows the channel name. + // shows the channel name. Signed out, the page on screen is the public landing whatever `page` + // says — the tab read "What should I watch?" over the sign-in form (prod UAT 2026-09-19). + const signedIn = useMe() != null; useEffect(() => { const brand = "Siftlode"; - const label = channelView - ? channelView.name || t("header.channelManager") - : page === "feed" - ? null - : t(pageTitleKey(page)); + const label = !signedIn + ? null + : channelView + ? channelView.name || t("header.channelManager") + : page === "feed" + ? null + : t(pageTitleKey(page)); document.title = label ? `${label} · ${brand}` : brand; - }, [page, channelView, i18n.language, t]); + }, [signedIn, page, channelView, i18n.language, t]); // In-app Back/Forward: stamp the initial entry with the current page (mirrored to the address bar), // then sync state from history.state on popstate — the browser restores each entry's URL for us, so diff --git a/frontend/src/components/legal/LegalLayout.tsx b/frontend/src/components/legal/LegalLayout.tsx index 5b0d30bd..e34b163a 100644 --- a/frontend/src/components/legal/LegalLayout.tsx +++ b/frontend/src/components/legal/LegalLayout.tsx @@ -71,6 +71,11 @@ export default function LegalLayout({ title, children }: { title: string; childr const id = decodeURIComponent(window.location.hash.slice(1)); if (id) document.getElementById(id)?.scrollIntoView(); }, []); + // The tab names the document, like every page of the app — a bookmark or a tab strip read just + // "Siftlode" (prod UAT 2026-09-19). + useEffect(() => { + document.title = `${title} · Siftlode`; + }, [title]); return (
diff --git a/frontend/src/i18n/locales/en/downloads.json b/frontend/src/i18n/locales/en/downloads.json index 1e750cba..adc16396 100644 --- a/frontend/src/i18n/locales/en/downloads.json +++ b/frontend/src/i18n/locales/en/downloads.json @@ -18,7 +18,7 @@ "page": { "title": "Downloads", "subtitle": "Download videos to the server, then save them to your device.", - "addUrl": "Paste a YouTube link or video id…", + "addUrl": "Paste a video link (YouTube, Facebook, …) or a YouTube video id…", "add": "Add", "paste": "Paste link from clipboard", "upload": "Upload", diff --git a/frontend/src/i18n/locales/hu/downloads.json b/frontend/src/i18n/locales/hu/downloads.json index 7c1ec57e..00362e97 100644 --- a/frontend/src/i18n/locales/hu/downloads.json +++ b/frontend/src/i18n/locales/hu/downloads.json @@ -18,7 +18,7 @@ "page": { "title": "Letöltések", "subtitle": "Töltsd le a videókat a szerverre, majd mentsd a saját gépedre.", - "addUrl": "Illessz be egy YouTube linket vagy videó azonosítót…", + "addUrl": "Illessz be egy videólinket (YouTube, Facebook, …) vagy YouTube-videóazonosítót…", "add": "Hozzáad", "paste": "Link beillesztése a vágólapról", "upload": "Feltöltés", From 582c033e1ccc7829736ab168fe4051d8a011195f Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 19 Sep 2026 04:51:43 +0200 Subject: [PATCH 07/21] feat(feed): the card menu carries the card's own actions Right-click, long-press or Shift+F10 on a feed card offered only the comfort-shelf pin; watched, saved, hidden and the progress reset were reachable only by hovering or Tabbing through every card. --- frontend/src/components/ComfortPinMenu.tsx | 67 ++++++++- .../src/components/FeedVideoMenu.dom.test.tsx | 134 ++++++++++++++++++ frontend/src/components/VirtualFeed.tsx | 13 +- frontend/src/i18n/locales/en/card.json | 12 +- frontend/src/i18n/locales/hu/card.json | 12 +- 5 files changed, 228 insertions(+), 10 deletions(-) create mode 100644 frontend/src/components/FeedVideoMenu.dom.test.tsx diff --git a/frontend/src/components/ComfortPinMenu.tsx b/frontend/src/components/ComfortPinMenu.tsx index 7a3c5e73..43e62256 100644 --- a/frontend/src/components/ComfortPinMenu.tsx +++ b/frontend/src/components/ComfortPinMenu.tsx @@ -1,7 +1,9 @@ import { useTranslation } from "react-i18next"; +import { Bookmark, BookmarkX, Check, Eye, EyeOff, RotateCcw, Undo2 } from "lucide-react"; -import { type PlexCard, type Video } from "../lib/api"; +import { type PlexCard, type Video, type VideoStatus } from "../lib/api"; import { pinPlex, pinVideo, useComfortPin, type PinTarget } from "./comfortPin"; +import { type ActionMenuItem } from "./ui/ActionMenu"; import { ContextMenu } from "./ui/ContextMenu"; import { type ContextMenuController } from "./ui/useContextMenu"; @@ -44,7 +46,64 @@ export function PlexPinMenu({ menu }: { menu: ContextMenuController }) return c.title} />; } -/** The YouTube video lists (the feed grid, the Home rails). */ -export function VideoPinMenu({ menu }: { menu: ContextMenuController