peter
5229fd96be
Channels, search and sync do no database work in their routes any more: reads go through catalog/queries, writes through catalog/service, and a live search's ingest is catalog/search. What a catalog route needs from Reader (a playlist's channels, a user's tag links, the feed projection of results) it asks reader/queries and reader/service. The scheduler and the runs page read channel titles and the work queue through catalog/queries. Catalog is fenced below its HTTP layer: its owner filters go through userscope, it reads no other unit's tables, and its ingest jobs commit only where they are named.
296 lines
11 KiB
Python
296 lines
11 KiB
Python
"""Catalog's service and queries, one use case each, below the HTTP layer.
|
|
|
|
The routes are thin callers now, and their own tests stay the contract a client sees. These hold
|
|
what each write does and leaves on its own: that it flushes and leaves the commit to its caller,
|
|
that a row of someone else's reads as a missing one, and the rule each write carries (an explored
|
|
channel flagged only while nobody follows it, a subscription that keeps it, a cleared search that
|
|
spares what anyone holds). DB-backed (real Postgres).
|
|
"""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import select
|
|
|
|
from app.catalog import queries, search, service
|
|
from app.catalog.models import (
|
|
BlockedChannel,
|
|
Channel,
|
|
ExploredChannel,
|
|
SearchFind,
|
|
Subscription,
|
|
Video,
|
|
)
|
|
from app.reader import queries as reader_queries
|
|
from app.reader import service as reader_service
|
|
|
|
|
|
def _channel(db, cid="UCc", **kw):
|
|
ch = Channel(id=cid, title=kw.pop("title", cid), **kw)
|
|
db.add(ch)
|
|
db.commit()
|
|
return ch
|
|
|
|
|
|
def _video(db, vid, cid="UCc", **kw):
|
|
v = Video(id=vid, channel_id=cid, published_at=datetime.now(timezone.utc), **kw)
|
|
db.add(v)
|
|
db.commit()
|
|
return v
|
|
|
|
|
|
def _sub(db, user, cid="UCc", **kw):
|
|
s = Subscription(user_id=user.id, channel_id=cid, **kw)
|
|
db.add(s)
|
|
db.commit()
|
|
return s
|
|
|
|
|
|
def _uncommitted(db):
|
|
return bool(db.new or db.dirty or db.deleted) or db.in_transaction()
|
|
|
|
|
|
# --- whose a row is ------------------------------------------------------------------------
|
|
|
|
|
|
def test_a_users_subscriptions_blocks_and_explorations_are_theirs_alone(db, make_user):
|
|
owner, stranger = make_user(), make_user()
|
|
ch = _channel(db)
|
|
_sub(db, owner)
|
|
service.explore(db, owner, ch)
|
|
service.block(db, owner, "UCc")
|
|
db.commit()
|
|
assert queries.subscription(db, owner, "UCc") is not None
|
|
assert queries.subscription(db, stranger, "UCc") is None
|
|
assert queries.exploration(db, stranger, "UCc") is None
|
|
assert queries.is_blocked(db, stranger, "UCc") is False
|
|
assert queries.blocked_ids(db, stranger) == set()
|
|
assert [s.channel_id for s, _ in queries.subscriptions(db, stranger, "en")] == []
|
|
|
|
|
|
# --- subscription settings -------------------------------------------------------------------
|
|
|
|
|
|
def test_full_history_reports_only_the_switch_on(db, make_user):
|
|
user = make_user()
|
|
_channel(db)
|
|
sub = _sub(db, user)
|
|
assert service.update_subscription(db, sub, priority=3) is False
|
|
assert service.update_subscription(db, sub, deep_requested=True) is True
|
|
assert service.update_subscription(db, sub, deep_requested=True) is False # already on
|
|
assert _uncommitted(db)
|
|
db.commit()
|
|
assert (sub.priority, sub.hidden, sub.deep_requested) == (3, False, True)
|
|
|
|
|
|
def test_full_history_for_all_moves_only_the_callers_rows(db, make_user):
|
|
user, other = make_user(), make_user()
|
|
_channel(db)
|
|
_sub(db, user)
|
|
theirs = _sub(db, other)
|
|
assert service.set_deep_all(db, user, True) == 1
|
|
db.commit()
|
|
db.expire_all()
|
|
assert theirs.deep_requested is False
|
|
|
|
|
|
def test_a_reset_asks_every_subscriber_for_full_history(db, make_user):
|
|
a, b = make_user(), make_user()
|
|
ch = _channel(db, backfill_done=True, recent_synced_at=datetime.now(timezone.utc))
|
|
_sub(db, a)
|
|
_sub(db, b)
|
|
service.reset_backfill(db, ch)
|
|
db.commit()
|
|
db.expire_all()
|
|
assert (ch.backfill_done, ch.recent_synced_at) == (False, None)
|
|
assert set(db.scalars(select(Subscription.deep_requested))) == {True}
|
|
|
|
|
|
# --- explore, subscribe, block ---------------------------------------------------------------
|
|
|
|
|
|
def test_an_exploration_is_one_row_and_flags_the_channel_only_while_nobody_follows_it(
|
|
db, make_user
|
|
):
|
|
explorer, follower = make_user(), make_user()
|
|
lone = _channel(db, "UClone")
|
|
kept = _channel(db, "UCkept")
|
|
_sub(db, follower, "UCkept")
|
|
service.explore(db, explorer, lone)
|
|
service.explore(db, explorer, kept)
|
|
db.commit()
|
|
first = queries.exploration(db, explorer, "UClone").created_at
|
|
service.explore(db, explorer, lone) # refreshes the grace clock
|
|
db.commit()
|
|
assert db.scalar(select(ExploredChannel.id).where(ExploredChannel.channel_id == "UClone"))
|
|
assert len(db.scalars(select(ExploredChannel.id)).all()) == 2
|
|
assert queries.exploration(db, explorer, "UClone").created_at >= first
|
|
assert (lone.from_explore, kept.from_explore) == (True, False)
|
|
|
|
|
|
def test_subscribing_keeps_an_explored_channel_and_its_videos(db, make_user):
|
|
user = make_user()
|
|
ch = _channel(db, from_explore=True)
|
|
v = _video(db, "v1", via_explore=True)
|
|
service.record_subscription(db, user, ch, "")
|
|
db.commit()
|
|
db.expire_all()
|
|
sub = queries.subscription(db, user, "UCc")
|
|
assert (sub.yt_subscription_id, ch.from_explore, v.via_explore) == (None, False, False)
|
|
service.drop_subscription(db, sub)
|
|
db.commit()
|
|
assert queries.subscription(db, user, "UCc") is None
|
|
|
|
|
|
def test_a_block_is_once_and_ends_the_exploration(db, make_user):
|
|
user = make_user()
|
|
ch = _channel(db)
|
|
service.explore(db, user, ch)
|
|
db.commit()
|
|
service.block(db, user, "UCc")
|
|
service.block(db, user, "UCc")
|
|
db.commit()
|
|
assert len(db.scalars(select(BlockedChannel.id)).all()) == 1
|
|
assert queries.exploration(db, user, "UCc") is None
|
|
assert [c.id for c in queries.blocked_channels(db, user, "en")] == ["UCc"]
|
|
service.unblock(db, user, "UCc")
|
|
db.commit()
|
|
assert queries.is_blocked(db, user, "UCc") is False
|
|
|
|
|
|
# --- what the status cards count -------------------------------------------------------------
|
|
|
|
|
|
def test_my_progress_counts_only_my_channels_and_my_unhidden_videos(db, make_user):
|
|
user, other = make_user(), make_user()
|
|
now = datetime.now(timezone.utc)
|
|
_channel(db, "UCa", details_synced_at=now, recent_synced_at=now, backfill_done=True)
|
|
_channel(db, "UCb")
|
|
_channel(db, "UCtheirs")
|
|
_video(db, "a1", "UCa")
|
|
_video(db, "b1", "UCb")
|
|
_video(db, "t1", "UCtheirs")
|
|
_sub(db, user, "UCa", deep_requested=True)
|
|
_sub(db, user, "UCb", hidden=True)
|
|
_sub(db, other, "UCtheirs")
|
|
assert queries.my_progress(db, user) == (2, 1, 1, 1)
|
|
assert [c.id for c in queries.my_deep_requested(db, user)] == ["UCa"]
|
|
assert queries.my_video_count(db, user) == 1
|
|
assert queries.video_total(db) == 3
|
|
assert queries.catalog_totals(db, user)["subscriptions"] == 2
|
|
|
|
|
|
def test_a_channel_with_no_title_is_left_out_of_the_titles(db):
|
|
_channel(db, "UCt", title="Titled")
|
|
db.add(Channel(id="UCn", title=None))
|
|
db.commit()
|
|
assert queries.channel_titles(db, {"UCt", "UCn", "UCgone"}) == {"UCt": "Titled"}
|
|
assert queries.channel_title(db, "UCt") == "Titled"
|
|
|
|
|
|
def test_the_deep_backlog_is_the_unfinished_channels_someone_asked_for(db, make_user):
|
|
user = make_user()
|
|
_channel(db, "UCwant")
|
|
_channel(db, "UCdone", backfill_done=True)
|
|
_channel(db, "UCnobody")
|
|
_sub(db, user, "UCwant", deep_requested=True)
|
|
_sub(db, user, "UCdone", deep_requested=True)
|
|
_sub(db, user, "UCnobody")
|
|
assert [c.id for c in queries.deep_backlog(db)] == ["UCwant"]
|
|
assert queries.deep_queued(db, ["UCwant", "UCdone", "UCnobody"]) == {"UCwant", "UCdone"}
|
|
|
|
|
|
# --- live search -----------------------------------------------------------------------------
|
|
|
|
|
|
def test_a_cleared_search_spares_what_anyone_holds(db, make_user):
|
|
user, other, follower = make_user(), make_user(), make_user()
|
|
_channel(db, "UCs")
|
|
_channel(db, "UCfollowed")
|
|
_sub(db, follower, "UCfollowed")
|
|
for vid in ("loose", "watched", "found_by_other"):
|
|
_video(db, vid, "UCs", via_search=True)
|
|
_video(db, "followed", "UCfollowed", via_search=True)
|
|
_video(db, "organic", "UCs")
|
|
ids = ["loose", "watched", "found_by_other", "followed", "organic"]
|
|
search.record_finds(db, user, ids, "cat")
|
|
search.record_finds(db, user, ids, "cat") # idempotent
|
|
search.record_finds(db, other, ["found_by_other"], "dog")
|
|
reader_service.mark(db, other, "watched", "watched")
|
|
db.commit()
|
|
assert search.clear(db, user, ids) == 1
|
|
db.commit()
|
|
left = set(db.scalars(select(Video.id)))
|
|
assert left == {"watched", "found_by_other", "followed", "organic"}
|
|
assert db.scalars(select(SearchFind.user_id)).all() == [other.id]
|
|
|
|
|
|
def test_a_search_puts_what_the_user_already_has_last(db, make_user):
|
|
user = make_user()
|
|
_channel(db, "UCs")
|
|
_channel(db, "UCmine")
|
|
_sub(db, user, "UCmine")
|
|
for vid, cid in (("new1", "UCs"), ("mine", "UCmine"), ("seen", "UCs"), ("new2", "UCs")):
|
|
_video(db, vid, cid)
|
|
reader_service.mark(db, user, "seen", "hidden")
|
|
db.commit()
|
|
assert search.new_first(db, user, ["mine", "new1", "seen", "new2"]) == [
|
|
"new1",
|
|
"new2",
|
|
"mine",
|
|
"seen",
|
|
]
|
|
|
|
|
|
# --- what Catalog asks Reader ----------------------------------------------------------------
|
|
|
|
|
|
def test_a_users_tag_links_are_made_once_and_only_theirs_are_read(db, make_user):
|
|
user, other = make_user(), make_user()
|
|
_channel(db)
|
|
mine = reader_service.create_tag(db, user, "Mine", "other", None)
|
|
theirs = reader_service.create_tag(db, other, "Theirs", "other", None)
|
|
reader_service.attach_user_tag(db, user, "UCc", mine.id)
|
|
reader_service.attach_user_tag(db, user, "UCc", mine.id)
|
|
reader_service.attach_user_tag(db, other, "UCc", theirs.id)
|
|
db.commit()
|
|
assert reader_queries.user_tag_links(db, user) == {"UCc": [mine.id]}
|
|
assert reader_queries.user_tag_links(db, user, "UCelse") == {}
|
|
reader_service.detach_user_tag(db, user, "UCc", mine.id)
|
|
db.commit()
|
|
assert reader_queries.user_tag_links(db, user) == {}
|
|
assert reader_queries.user_tag_links(db, other) == {"UCc": [theirs.id]}
|
|
|
|
|
|
def test_discovery_offers_the_playlist_channels_not_followed_and_not_ones_own(db, make_user):
|
|
user = make_user()
|
|
for cid in ("UCnew", "UCfollowed", "UCown"):
|
|
_channel(db, cid)
|
|
for vid, cid in (("n1", "UCnew"), ("n2", "UCnew"), ("f1", "UCfollowed"), ("o1", "UCown")):
|
|
_video(db, vid, cid)
|
|
_sub(db, user, "UCfollowed")
|
|
pl = reader_service.create_playlist(db, user, "L")
|
|
for vid in ("n1", "n2", "f1", "o1"):
|
|
reader_service.add_item(db, pl, vid, mark_dirty=False)
|
|
db.commit()
|
|
rows = reader_queries.playlist_channels(db, user, "UCown", "en")
|
|
assert [(ch.id, n, lists) for ch, n, lists in rows] == [("UCnew", 2, 1)]
|
|
|
|
|
|
def test_an_exploration_older_than_its_grace_is_still_one_row(db, make_user):
|
|
"""The grace clock is refreshed in place: re-exploring after the cutoff does not add a row."""
|
|
user = make_user()
|
|
ch = _channel(db)
|
|
db.add(
|
|
ExploredChannel(
|
|
user_id=user.id,
|
|
channel_id="UCc",
|
|
created_at=datetime.now(timezone.utc) - timedelta(days=30),
|
|
)
|
|
)
|
|
db.commit()
|
|
service.explore(db, user, ch)
|
|
db.commit()
|
|
rows = db.scalars(select(ExploredChannel)).all()
|
|
assert len(rows) == 1
|
|
assert rows[0].created_at > datetime.now(timezone.utc) - timedelta(minutes=1)
|