refactor(catalog): the HTTP layer reads through queries and writes through the service

Channels, search and sync do no database work in their routes any more: reads go through
catalog/queries, writes through catalog/service, and a live search's ingest is catalog/search.
What a catalog route needs from Reader (a playlist's channels, a user's tag links, the feed
projection of results) it asks reader/queries and reader/service. The scheduler and the runs page
read channel titles and the work queue through catalog/queries. Catalog is fenced below its HTTP
layer: its owner filters go through userscope, it reads no other unit's tables, and its ingest
jobs commit only where they are named.
This commit is contained in:
2026-10-19 02:32:20 +02:00
parent 238e6f8a61
commit 5229fd96be
21 changed files with 1223 additions and 726 deletions
+42 -8
View File
@@ -110,7 +110,6 @@ UNIT_ROUTES = {
# What each HTTP file still does, by file: the debt a unit's sprint pays off. Exact counts. A core
# route's count is only the unit tables it imports; it is paid by the sprint of the unit it reaches.
ALLOWED = {
"channels.py": 83,
"plex/_shared.py": 1,
"plex/admin.py": 12,
"plex/browse.py": 93,
@@ -120,10 +119,6 @@ ALLOWED = {
"plex/people.py": 20,
"plex/playback.py": 42,
"plex/playlists.py": 80,
"runs.py": 1,
"scheduler.py": 3,
"search.py": 46,
"sync.py": 38,
}
# The session methods that read or write data. `commit` and `rollback` stay with the handler.
@@ -341,11 +336,22 @@ UNIT_PACKAGES = {
"sync/playlist_merge.py",
"sync/tagsuggest.py",
),
# The catalog's ingest jobs: the subscription import, the video sync and backfills, the explore
# and search ingests, the maintenance re-checks and the auto-tagger.
"catalog": (
"catalog/",
"sync/autotag.py",
"sync/explore.py",
"sync/maintenance.py",
"sync/runner.py",
"sync/subscriptions.py",
"sync/videos.py",
),
}
# The units fenced so far; the rest join with their sprint.
FENCED = {"capture", "reader"}
FENCED = {"capture", "reader", "catalog"}
# Whose tables a unit's own code may READ. Core code reads a fenced unit's tables through its queries.
READS = {"capture": {"catalog", "plex"}, "reader": {"catalog", "plex"}}
READS = {"capture": {"catalog", "plex"}, "reader": {"catalog", "plex"}, "catalog": set()}
# The columns that say whose a row is.
OWNER_COLUMNS = {"user_id", "to_user_id", "from_user_id", "created_by"}
# Where a fenced unit's code may commit: the jobs that own their session (the worker and what it
@@ -372,6 +378,32 @@ UNIT_COMMITS = {
"sync/playlists.py:merge_playlist",
"sync/playlists.py:repull_playlist",
},
# The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page
# fetched and stored is not lost when the next call fails, and an enrich that fails rolls back
# to the stubs the step before committed (a search's results, a playlist's unseen videos).
"catalog": {
"catalog/search.py:classify_shorts",
"catalog/search.py:ingest_candidates",
"sync/autotag.py:apply_channel_autotags",
"sync/autotag.py:run_autotag_all",
"sync/explore.py:_enrich_new",
"sync/explore.py:purge_unkept_explored",
"sync/explore.py:purge_unkept_search",
"sync/maintenance.py:_notify_removed",
"sync/maintenance.py:_recheck_flagged",
"sync/maintenance.py:_revalidate_rolling",
"sync/runner.py:run_feed_fallback",
"sync/runner.py:_stamp_fallback_attempt",
"sync/subscriptions.py:import_subscriptions",
"sync/videos.py:insert_stubs",
"sync/videos.py:apply_rss_feed",
"sync/videos.py:backfill_channel_recent",
"sync/videos.py:backfill_channel_deep",
"sync/videos.py:reconcile_full_history",
"sync/videos.py:_apply_video_batch",
"sync/videos.py:run_shorts_classification",
"sync/videos.py:ensure_videos",
},
}
SQL_WRITES = {"insert", "update", "delete", "pg_insert"}
@@ -506,7 +538,9 @@ PACKAGE_CASES = {
("downloads/x.py", "from app.models import Video\nupdate(Video)"): 1,
("downloads/x.py", "from app.downloads.models import DownloadJob\nDownloadJob()"): 0,
("search.py", "from app.downloads.models import DownloadJob"): 1, # core reads capture
("search.py", "from app.models import Video\nVideo()"): 0, # neither side fenced yet
("search.py", "from app.models import PlexItem\nPlexItem()"): 0, # neither side fenced yet
("search.py", "from app.models import Video\nVideo()"): 2, # core reads and writes catalog
("sync/videos.py", "from app.models import Playlist"): 1, # no catalog→reader edge
("downloads/x.py", "q = DownloadJob.user_id == 1"): 1,
("downloads/x.py", "q = owned_by(DownloadJob.user_id, 1)"): 0,
("downloads/x.py", "def f(db):\n db.commit()"): 1,