Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e2a8503496 | ||
|
|
a389306885 | ||
|
|
32c05897f4 | ||
|
|
2e319be569 | ||
|
|
852b227ea2 | ||
|
|
06b85586e6 | ||
|
|
fc32188fd2 | ||
|
|
9538fb1b7a | ||
|
|
5051809940 | ||
|
|
64dde6fd36 | ||
|
|
38921fb997 | ||
|
|
2d1a7ff374 | ||
|
|
361ca7872e | ||
|
|
81b7660232 | ||
|
|
7caa2636e4 | ||
|
|
1e894d378d | ||
|
|
6fb369ffcc | ||
|
|
a3500340cf | ||
|
|
b1832afd18 | ||
|
|
434a98d6ed | ||
|
|
cc1f196a2c | ||
|
|
e3dac34bb0 | ||
|
|
938135bb59 | ||
|
|
7d2bee59a3 | ||
|
|
6af238c2ea | ||
|
|
b71b33654a | ||
|
|
5d99a5f253 | ||
|
|
c0d869089d | ||
|
|
15f5a5694a | ||
|
|
14a44bff6e | ||
|
|
37d11b602f | ||
|
|
da3cb5e694 | ||
|
|
1e5232f47a | ||
|
|
dddaeb9a38 | ||
|
|
70a74f8629 | ||
|
|
8f1309fa18 | ||
|
|
b7a4eaff8d | ||
|
|
a20e27800c | ||
|
|
fa7a09a420 | ||
|
|
f983a491c0 | ||
|
|
f85dc12819 | ||
|
|
9798e506cf | ||
|
|
a60164edc4 | ||
|
|
6c7038cf1f | ||
|
|
f8e144a658 | ||
|
|
b4984dc38b | ||
|
|
21c9b34540 | ||
|
|
041cd526b7 | ||
|
|
9477fea3dc | ||
|
|
e4b19b5320 | ||
|
|
7692253636 | ||
|
|
e90603d4db | ||
|
|
dd76e9dcf7 | ||
|
|
fca989c098 | ||
|
|
7ebbf45921 | ||
|
|
829cadf5fb | ||
|
|
8f08073564 | ||
|
|
af08a7850b | ||
|
|
d2193ccd1b | ||
|
|
cda70164ba | ||
|
|
5e1cc4d981 | ||
|
|
4ee452b854 | ||
|
|
edbc9dede5 | ||
|
|
53ca4e2849 | ||
|
|
1801a58f6b | ||
|
|
011acd919e | ||
|
|
45d8b47366 | ||
|
|
c3820a0b44 | ||
|
|
c7b393901f | ||
|
|
f1e14c94c5 | ||
|
|
ddb171fca3 | ||
|
|
ec3bbc78dc | ||
|
|
1be1dc9d06 | ||
|
|
6109e0c6f1 | ||
|
|
5851ddd329 | ||
|
|
ad0cf5827b | ||
|
|
f6192aa6bf | ||
|
|
6b4febcc95 | ||
|
|
cb5ac09a80 | ||
|
|
bad2aae753 | ||
|
|
8f6714e678 | ||
|
|
13fb2b4e61 | ||
|
|
5316f48af2 | ||
|
|
0c746f5bcd | ||
|
|
caced8664c | ||
|
|
29f2c9059e | ||
|
|
b83248c510 | ||
|
|
2b5de89eda | ||
|
|
ff45db70ae | ||
|
|
7d40a8780d | ||
|
|
d56a791c04 | ||
|
|
187b8007f8 | ||
|
|
f10915c223 | ||
|
|
f9b63745f9 | ||
|
|
5b5e76b73d | ||
|
|
91be3d0535 | ||
|
|
3bde114b14 | ||
|
|
fca3fa13ef | ||
|
|
df60bc5ff5 | ||
|
|
99d63d272e | ||
|
|
482dd28006 | ||
|
|
0b7180ce8f | ||
|
|
d2f2085f1f | ||
|
|
062015ba27 | ||
|
|
0401dae689 | ||
|
|
e8a1c252a9 | ||
|
|
fa8bdf83a8 | ||
|
|
868ff533ee | ||
|
|
00bf9a39f7 | ||
|
|
640a8efeed | ||
|
|
23ac884b42 | ||
|
|
1579a81395 | ||
|
|
4fe6f38f1b | ||
|
|
94a608053a | ||
|
|
aea99cc71a | ||
|
|
180ab56379 | ||
|
|
0b592345e6 | ||
|
|
4800df4208 | ||
|
|
8bfc01684f | ||
|
|
48aa1a25f5 | ||
|
|
c82cf25bcf | ||
|
|
634e9bb17c | ||
|
|
ce2954cf2b | ||
|
|
2229b01a91 | ||
|
|
b17b47f54e | ||
|
|
1e2b3c5d29 | ||
|
|
d065902404 | ||
|
|
a300977ed4 | ||
|
|
ddb7818d38 | ||
|
|
48787af11c | ||
|
|
b733ee84fa | ||
|
|
96413c3d5e | ||
|
|
d9677e0e3a | ||
|
|
2b804300d5 | ||
|
|
3f047bbad1 | ||
|
|
8a97c79d37 | ||
|
|
d37b6ef07f | ||
|
|
88c5d5c5cb | ||
|
|
cff8bed62b | ||
|
|
bc79720e81 | ||
|
|
b02a9e37e8 | ||
|
|
b2013a079a | ||
|
|
0fab0ce779 | ||
|
|
4ace486618 | ||
|
|
3334a8e895 | ||
|
|
e2362e1ed1 | ||
|
|
7d89bc8b8e | ||
|
|
f0a0214d97 | ||
|
|
e824fe0ae9 | ||
|
|
49011940d6 | ||
|
|
52b6535dc6 | ||
|
|
9bf7da2cbe | ||
|
|
3301976375 | ||
|
|
fa176a3e6a | ||
|
|
e8771f4d6b | ||
|
|
94dbe4b228 | ||
|
|
0ed4ac6958 | ||
|
|
8b8b0ca5ab | ||
|
|
6116ad4bf4 | ||
|
|
943143ef45 | ||
|
|
21a3aae700 | ||
|
|
6a35e83ded | ||
|
|
8c08ea3133 | ||
|
|
3af3799b25 |
@@ -4,3 +4,6 @@
|
|||||||
#
|
#
|
||||||
# Prettier one-time repo-wide format pass (R2 S1b, 2026-07-21)
|
# Prettier one-time repo-wide format pass (R2 S1b, 2026-07-21)
|
||||||
51c36c2071728d2cd7ae580b7bfdf21f6867a347
|
51c36c2071728d2cd7ae580b7bfdf21f6867a347
|
||||||
|
|
||||||
|
# ruff format one-time backend + scripts pass (line-length 100, 2026-08-23)
|
||||||
|
b1832afd180866b922521148603db447020c57a4
|
||||||
|
|||||||
@@ -59,6 +59,8 @@ jobs:
|
|||||||
- run: pip install --no-cache-dir -r backend/requirements.lock -r backend/requirements-dev.txt
|
- run: pip install --no-cache-dir -r backend/requirements.lock -r backend/requirements-dev.txt
|
||||||
- name: ruff
|
- name: ruff
|
||||||
run: python -m ruff check --config backend/ruff.toml backend scripts
|
run: python -m ruff check --config backend/ruff.toml backend scripts
|
||||||
|
- name: ruff format --check
|
||||||
|
run: python -m ruff format --check --config backend/ruff.toml backend scripts
|
||||||
- name: pytest
|
- name: pytest
|
||||||
run: python -m pytest
|
run: python -m pytest
|
||||||
working-directory: backend
|
working-directory: backend
|
||||||
|
|||||||
@@ -12,12 +12,12 @@ provider.
|
|||||||
|
|
||||||
## The gate — never merge or ship red
|
## The gate — never merge or ship red
|
||||||
|
|
||||||
The local gate is **ruff + typecheck (3 tsc projects) + eslint + prettier --check + knip + vitest +
|
The local gate is **ruff (`check` + `format --check`) + typecheck (3 tsc projects) + eslint + prettier
|
||||||
pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
--check + knip + vitest + pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
||||||
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
||||||
|
|
||||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||||
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` and `pytest` (the DB-lane needs Postgres).
|
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` + `ruff format --check backend` and `pytest` (the DB-lane needs Postgres).
|
||||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||||
|
|
||||||
## Workflow
|
## Workflow
|
||||||
@@ -26,12 +26,33 @@ pytest**. It runs on every `publish`, which **refuses to build on failure — th
|
|||||||
keep commits logical and Conventional-Commit-styled.
|
keep commits logical and Conventional-Commit-styled.
|
||||||
- **No AI attribution in commits.** No `Co-Authored-By` trailer, no "generated by" lines. (This file
|
- **No AI attribution in commits.** No `Co-Authored-By` trailer, no "generated by" lines. (This file
|
||||||
being AI-written is the intended, visible exception; commits are not.)
|
being AI-written is the intended, visible exception; commits are not.)
|
||||||
- **Release is a deliberate, user-owned step** (`siftlode release` from the feature branch: merge to
|
- **Release: user-owned in timing, agent-run in execution.** The user decides *when* to ship; on their
|
||||||
`main` → publish → deploy → verify `/api/version`). Do not self-ship; bring work to a green,
|
go-ahead *you* run `siftlode release` from the feature branch (merge to `main` → publish → deploy →
|
||||||
committed state and stop for the user's go-ahead.
|
verify `/api/version`). Never self-ship — bring work to a green, committed state and stop for the
|
||||||
|
go-ahead first, then run it yourself.
|
||||||
- **Public repo.** Keep ops/topology/secrets out — those live in a separate private repo. Nothing here
|
- **Public repo.** Keep ops/topology/secrets out — those live in a separate private repo. Nothing here
|
||||||
should assume or leak them.
|
should assume or leak them.
|
||||||
|
|
||||||
|
## Collaboration — how to work with the human
|
||||||
|
|
||||||
|
The human is the product owner: they bring ideas and requirements. You are the technical lead — you own
|
||||||
|
how it gets built. Concretely:
|
||||||
|
|
||||||
|
- **Plan before you build.** For anything non-trivial, present a short plan (sprint-level for larger
|
||||||
|
work) and get a go-ahead before writing code.
|
||||||
|
- **Offer choices, mark the optimal — and explain the trade-off.** When several solutions are viable,
|
||||||
|
list them and mark the recommended one. This is the *one* place to spend extra words: give the *why*
|
||||||
|
behind each option and the weight of the decision. These decision points are where the product owner
|
||||||
|
engages with the build and learns from it, so make them understandable — a reasoned menu, not a bare one.
|
||||||
|
- **Mockups for design/UX decisions.** For any visual or interaction choice, show mockup options to
|
||||||
|
choose from before building.
|
||||||
|
- **Challenge unsound requests.** If a request is technically ill-advised and you can back it with real
|
||||||
|
arguments, push back or propose an alternative — don't comply blindly.
|
||||||
|
- **Verify UI in the browser before calling it done.** tsc/unit/DOM tests miss visual and interaction
|
||||||
|
bugs; do a real visual check of any UI change.
|
||||||
|
- **Be terse everywhere else.** Outside those decision points, lead with the answer or decision. During
|
||||||
|
implementation and debugging especially: no restating the same point reworded, no filler.
|
||||||
|
|
||||||
## Architecture shape (don't fight it)
|
## Architecture shape (don't fight it)
|
||||||
|
|
||||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||||
@@ -79,5 +100,6 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
*A fuller written-conventions doc (`docs/development.md`) is planned; once it lands, this file slims to a
|
*Run-it-locally and the full gate commands live in [`docs/development.md`](docs/development.md); backend
|
||||||
pointer plus the gate/workflow/no-attribution rules. Until then, the table above is the source of truth.*
|
code conventions in [`backend/app/README.md`](backend/app/README.md). This file stays the source of
|
||||||
|
truth for the canonical-patterns table and the gate/workflow/collaboration rules above.*
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0001_initial
|
|||||||
Revises:
|
Revises:
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0002_ingest_core
|
|||||||
Revises: 0001_initial
|
Revises: 0001_initial
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0003_tags
|
|||||||
Revises: 0002_ingest_core
|
Revises: 0002_ingest_core
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0004_feed_state_prefs
|
|||||||
Revises: 0003_tags
|
Revises: 0003_tags
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0005_shorts_probed
|
|||||||
Revises: 0004_feed_state_prefs
|
Revises: 0004_feed_state_prefs
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0006_app_state
|
|||||||
Revises: 0005_shorts_probed
|
Revises: 0005_shorts_probed
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0007_deep_requested
|
|||||||
Revises: 0006_app_state
|
Revises: 0006_app_state
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0008_invites
|
|||||||
Revises: 0007_deep_requested
|
Revises: 0007_deep_requested
|
||||||
Create Date: 2026-06-12
|
Create Date: 2026-06-12
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -22,9 +23,7 @@ def upgrade() -> None:
|
|||||||
"invites",
|
"invites",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("email", sa.String(length=320), nullable=False),
|
sa.Column("email", sa.String(length=320), nullable=False),
|
||||||
sa.Column(
|
sa.Column("status", sa.String(length=16), nullable=False, server_default="pending"),
|
||||||
"status", sa.String(length=16), nullable=False, server_default="pending"
|
|
||||||
),
|
|
||||||
sa.Column("note", sa.Text(), nullable=True),
|
sa.Column("note", sa.Text(), nullable=True),
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"requested_at",
|
"requested_at",
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0009_quota_events
|
|||||||
Revises: 0008_invites
|
Revises: 0008_invites
|
||||||
Create Date: 2026-06-12
|
Create Date: 2026-06-12
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0010_watch_progress
|
|||||||
Revises: 0009_quota_events
|
Revises: 0009_quota_events
|
||||||
Create Date: 2026-06-14
|
Create Date: 2026-06-14
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0011_playlists
|
|||||||
Revises: 0010_watch_progress
|
Revises: 0010_watch_progress
|
||||||
Create Date: 2026-06-15
|
Create Date: 2026-06-15
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -26,16 +27,10 @@ def upgrade() -> None:
|
|||||||
nullable=False,
|
nullable=False,
|
||||||
),
|
),
|
||||||
sa.Column("name", sa.String(length=255), nullable=False),
|
sa.Column("name", sa.String(length=255), nullable=False),
|
||||||
sa.Column(
|
sa.Column("kind", sa.String(length=16), nullable=False, server_default="user"),
|
||||||
"kind", sa.String(length=16), nullable=False, server_default="user"
|
sa.Column("source", sa.String(length=16), nullable=False, server_default="local"),
|
||||||
),
|
|
||||||
sa.Column(
|
|
||||||
"source", sa.String(length=16), nullable=False, server_default="local"
|
|
||||||
),
|
|
||||||
sa.Column("yt_playlist_id", sa.String(length=64), nullable=True),
|
sa.Column("yt_playlist_id", sa.String(length=64), nullable=True),
|
||||||
sa.Column(
|
sa.Column("dirty", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
"dirty", sa.Boolean(), nullable=False, server_default="false"
|
|
||||||
),
|
|
||||||
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"created_at",
|
"created_at",
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ Create Date: 2026-06-15
|
|||||||
Migrates the old per-video status='saved' into membership of a built-in 'watch_later'
|
Migrates the old per-video status='saved' into membership of a built-in 'watch_later'
|
||||||
playlist (one per user who had saved videos), then demotes those states to 'new'.
|
playlist (one per user who had saved videos), then demotes those states to 'new'.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ a YouTube read. Backfills the baseline for already-clean linked playlists (the c
|
|||||||
state == the synced state for them); dirty ones are left NULL so they stay dirty until the
|
state == the synced state for them); dirty ones are left NULL so they stay dirty until the
|
||||||
next real sync establishes a baseline.
|
next real sync establishes a baseline.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
@@ -32,24 +33,18 @@ def _fingerprint(name: str, ids: list[str]) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column(
|
op.add_column("playlists", sa.Column("synced_fingerprint", sa.String(), nullable=True))
|
||||||
"playlists", sa.Column("synced_fingerprint", sa.String(), nullable=True)
|
|
||||||
)
|
|
||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
# Backfill clean, YouTube-linked playlists: their current state IS the synced state.
|
# Backfill clean, YouTube-linked playlists: their current state IS the synced state.
|
||||||
rows = bind.execute(
|
rows = bind.execute(
|
||||||
sa.text(
|
sa.text("SELECT id, name FROM playlists WHERE yt_playlist_id IS NOT NULL AND dirty = false")
|
||||||
"SELECT id, name FROM playlists "
|
|
||||||
"WHERE yt_playlist_id IS NOT NULL AND dirty = false"
|
|
||||||
)
|
|
||||||
).fetchall()
|
).fetchall()
|
||||||
for pid, name in rows:
|
for pid, name in rows:
|
||||||
ids = [
|
ids = [
|
||||||
r[0]
|
r[0]
|
||||||
for r in bind.execute(
|
for r in bind.execute(
|
||||||
sa.text(
|
sa.text(
|
||||||
"SELECT video_id FROM playlist_items "
|
"SELECT video_id FROM playlist_items WHERE playlist_id = :pid ORDER BY position"
|
||||||
"WHERE playlist_id = :pid ORDER BY position"
|
|
||||||
),
|
),
|
||||||
{"pid": pid},
|
{"pid": pid},
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ Adds the shared demo-account plumbing:
|
|||||||
The demo user row itself is created lazily on first demo login (works across all three DBs
|
The demo user row itself is created lazily on first demo login (works across all three DBs
|
||||||
without a data migration), so this only adds the schema.
|
without a data migration), so this only adds the schema.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -47,9 +48,7 @@ def upgrade() -> None:
|
|||||||
nullable=False,
|
nullable=False,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
op.create_index(
|
op.create_index("ix_demo_whitelist_email", "demo_whitelist", ["email"], unique=True)
|
||||||
"ix_demo_whitelist_email", "demo_whitelist", ["email"], unique=True
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ Create Date: 2026-06-16
|
|||||||
Adds scheduler_settings: a per-job run-interval override (minutes), editable from the admin
|
Adds scheduler_settings: a per-job run-interval override (minutes), editable from the admin
|
||||||
Scheduler dashboard. Absent row = use the env/config default, so this is purely additive.
|
Scheduler dashboard. Absent row = use the env/config default, so this is purely additive.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ the validation lifecycle (last_checked_at for rolling re-validation; unavailable
|
|||||||
unavailable_reason to flag a video as currently unplayable — hidden from the feed
|
unavailable_reason to flag a video as currently unplayable — hidden from the feed
|
||||||
immediately, hard-deleted after a grace period). Purely additive; all columns nullable.
|
immediately, hard-deleted after a grace period). Purely additive; all columns nullable.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -26,7 +27,9 @@ def upgrade() -> None:
|
|||||||
op.add_column("videos", sa.Column("privacy_status", sa.String(length=16), nullable=True))
|
op.add_column("videos", sa.Column("privacy_status", sa.String(length=16), nullable=True))
|
||||||
op.add_column("videos", sa.Column("upload_status", sa.String(length=16), nullable=True))
|
op.add_column("videos", sa.Column("upload_status", sa.String(length=16), nullable=True))
|
||||||
op.add_column("videos", sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True))
|
op.add_column("videos", sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True))
|
||||||
op.add_column("videos", sa.Column("unavailable_since", sa.DateTime(timezone=True), nullable=True))
|
op.add_column(
|
||||||
|
"videos", sa.Column("unavailable_since", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
op.add_column("videos", sa.Column("unavailable_reason", sa.String(length=24), nullable=True))
|
op.add_column("videos", sa.Column("unavailable_reason", sa.String(length=24), nullable=True))
|
||||||
op.create_index("ix_videos_last_checked_at", "videos", ["last_checked_at"])
|
op.create_index("ix_videos_last_checked_at", "videos", ["last_checked_at"])
|
||||||
op.create_index("ix_videos_unavailable_since", "videos", ["unavailable_since"])
|
op.create_index("ix_videos_unavailable_since", "videos", ["unavailable_since"])
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ JSON payload column is named `data` (SQLAlchemy's declarative Base reserves `met
|
|||||||
First producer is the maintenance job's batched "videos removed" notice. Coexists with the
|
First producer is the maintenance job's batched "videos removed" notice. Coexists with the
|
||||||
existing client-side transient bell.
|
existing client-side transient bell.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -36,18 +37,14 @@ def upgrade() -> None:
|
|||||||
sa.Column("data", sa.JSON(), nullable=True),
|
sa.Column("data", sa.JSON(), nullable=True),
|
||||||
sa.Column("read", sa.Boolean(), nullable=False, server_default="false"),
|
sa.Column("read", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
sa.Column("dismissed", sa.Boolean(), nullable=False, server_default="false"),
|
sa.Column("dismissed", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
sa.Column(
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now()
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
op.create_index("ix_notifications_user_id", "notifications", ["user_id"])
|
op.create_index("ix_notifications_user_id", "notifications", ["user_id"])
|
||||||
op.create_index("ix_notifications_type", "notifications", ["type"])
|
op.create_index("ix_notifications_type", "notifications", ["type"])
|
||||||
op.create_index("ix_notifications_read", "notifications", ["read"])
|
op.create_index("ix_notifications_read", "notifications", ["read"])
|
||||||
op.create_index("ix_notifications_created_at", "notifications", ["created_at"])
|
op.create_index("ix_notifications_created_at", "notifications", ["created_at"])
|
||||||
# Fast unread-badge count + inbox listing per user.
|
# Fast unread-badge count + inbox listing per user.
|
||||||
op.create_index(
|
op.create_index("ix_notifications_user_read", "notifications", ["user_id", "read"])
|
||||||
"ix_notifications_user_read", "notifications", ["user_id", "read"]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds app_state.maintenance_revalidate_batch: an admin-tunable override (from the
|
|||||||
dashboard) for how many videos the maintenance job re-validates per run. NULL = use the
|
dashboard) for how many videos the maintenance job re-validates per run. NULL = use the
|
||||||
env/config default. Purely additive.
|
env/config default. Purely additive.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ channels.list?mine=true and cached. Used to exclude the user's own channel from
|
|||||||
playlist-discovery list (you don't subscribe to yourself). NULL = not resolved yet.
|
playlist-discovery list (you don't subscribe to yourself). NULL = not resolved yet.
|
||||||
Purely additive.
|
Purely additive.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ stored historical values to the canonical scheme defined in app.quota.QuotaActio
|
|||||||
fully reversible. (Scheduler job ids and progress phase labels are a separate namespace and
|
fully reversible. (Scheduler job ids and progress phase labels are a separate namespace and
|
||||||
are NOT touched.)
|
are NOT touched.)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds the system_config table: admin overrides for operational config that otherw
|
|||||||
from env/config defaults (see app.sysconfig for the registry of known keys). Secret values
|
from env/config defaults (see app.sysconfig for the registry of known keys). Secret values
|
||||||
are stored Fernet-encrypted (encrypted flag). Purely additive; absent rows = use defaults.
|
are stored Fernet-encrypted (encrypted flag). Purely additive; absent rows = use defaults.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ google_sub nullable (password-only accounts have no Google sub until they link),
|
|||||||
single-use auth_tokens table for email verification + password reset. Existing rows are all
|
single-use auth_tokens table for email verification + password reset. Existing rows are all
|
||||||
Google/demo accounts → backfilled email_verified=true (is_active defaults true).
|
Google/demo accounts → backfilled email_verified=true (is_active defaults true).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds users.is_suspended — an admin-controlled access block, distinct from is_a
|
|||||||
approval lifecycle). A suspended account can't sign in (password or Google) and any live
|
approval lifecycle). A suspended account can't sign in (password or Google) and any live
|
||||||
session is rejected; the user is told why and given the operator's contact.
|
session is rejected; the user is told why and given the operator's contact.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ email_verified=false (the callback didn't set it). Backfill those so the flag re
|
|||||||
matters because password-login gates on email_verified, so a Google user who later adds a password
|
matters because password-login gates on email_verified, so a Google user who later adds a password
|
||||||
must count as verified. Demo/password-only rows (google_sub IS NULL) are left untouched.
|
must count as verified. Demo/password-only rows (google_sub IS NULL) are left untouched.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds app_state.configured + setup_token_hash for the first-run install wizard. E
|
|||||||
(any users present) are marked configured so they never see the wizard; a genuinely fresh DB (no
|
(any users present) are marked configured so they never see the wizard; a genuinely fresh DB (no
|
||||||
users) starts unconfigured and boots into setup mode.
|
users) starts unconfigured and boots into setup mode.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds the `messages` table for direct user-to-user messaging (notification module
|
|||||||
A conversation is the set of rows between two users; both FKs cascade so a GDPR account
|
A conversation is the set of rows between two users; both FKs cascade so a GDPR account
|
||||||
erasure removes that user's messages.
|
erasure removes that user's messages.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ hold each user's public key + passphrase-wrapped private key (the server can nev
|
|||||||
Any pre-existing rows were plaintext with no keys, so they're dropped (the feature is unshipped;
|
Any pre-existing rows were plaintext with no keys, so they're dropped (the feature is unshipped;
|
||||||
prod has none).
|
prod has none).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ the Library by default:
|
|||||||
- channels.from_search — this channel exists only because search surfaced one of its videos
|
- channels.from_search — this channel exists only because search surfaced one of its videos
|
||||||
Both default false (every existing row is organic), indexed for the feed's exclude filter.
|
Both default false (every existing row is organic), indexed for the feed's exclude filter.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -25,16 +26,12 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column(
|
op.add_column(
|
||||||
"videos",
|
"videos",
|
||||||
sa.Column(
|
sa.Column("via_search", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
"via_search", sa.Boolean(), nullable=False, server_default="false"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
op.create_index("ix_videos_via_search", "videos", ["via_search"])
|
op.create_index("ix_videos_via_search", "videos", ["via_search"])
|
||||||
op.add_column(
|
op.add_column(
|
||||||
"channels",
|
"channels",
|
||||||
sa.Column(
|
sa.Column("from_search", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
"from_search", sa.Boolean(), nullable=False, server_default="false"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
op.create_index("ix_channels_from_search", "channels", ["from_search"])
|
op.create_index("ix_channels_from_search", "channels", ["from_search"])
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ query wraps both sides in unaccent(); this migration just makes the function ava
|
|||||||
`unaccent` ships with the official postgres image (postgresql-contrib). The DB role is a
|
`unaccent` ships with the official postgres image (postgresql-contrib). The DB role is a
|
||||||
superuser in our deployments, so CREATE EXTENSION succeeds; IF NOT EXISTS keeps it idempotent.
|
superuser in our deployments, so CREATE EXTENSION succeeds; IF NOT EXISTS keeps it idempotent.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ live YouTube search. This lets "videos I searched for" appear in that user's Min
|
|||||||
Mine Source filter), independent of the global `videos.via_search` flag (which marks a video as
|
Mine Source filter), independent of the global `videos.via_search` flag (which marks a video as
|
||||||
search-discovered by anyone, for the shared Library's Source filter).
|
search-discovered by anyone, for the shared Library's Source filter).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ search stays accent-insensitive ("tiesto" matches "Tiësto"). A GIN expression i
|
|||||||
query uses the exact same expression so the planner picks the index. `unaccent` is already
|
query uses the exact same expression so the planner picks the index. `unaccent` is already
|
||||||
enabled (migration 0029). The config-based 2-arg to_tsvector is IMMUTABLE, so it's index-safe.
|
enabled (migration 0029). The config-based 2-arg to_tsvector is IMMUTABLE, so it's index-safe.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -26,9 +27,7 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.execute(
|
op.execute("CREATE TEXT SEARCH CONFIGURATION public.unaccent_simple (COPY = pg_catalog.simple)")
|
||||||
"CREATE TEXT SEARCH CONFIGURATION public.unaccent_simple (COPY = pg_catalog.simple)"
|
|
||||||
)
|
|
||||||
op.execute(
|
op.execute(
|
||||||
"""
|
"""
|
||||||
ALTER TEXT SEARCH CONFIGURATION public.unaccent_simple
|
ALTER TEXT SEARCH CONFIGURATION public.unaccent_simple
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ title-only expression index from 0031 is dropped (superseded). Existing rows get
|
|||||||
description indexed immediately; keywords/search_terms fill in as videos are (re-)enriched and
|
description indexed immediately; keywords/search_terms fill in as videos are (re-)enriched and
|
||||||
searched. Uses the `unaccent_simple` config from 0031 (accent-insensitive).
|
searched. Uses the `unaccent_simple` config from 0031 (accent-insensitive).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -48,9 +49,7 @@ def upgrade() -> None:
|
|||||||
f"ALTER TABLE videos ADD COLUMN search_vector tsvector "
|
f"ALTER TABLE videos ADD COLUMN search_vector tsvector "
|
||||||
f"GENERATED ALWAYS AS ({_SEARCH_VECTOR_EXPR}) STORED"
|
f"GENERATED ALWAYS AS ({_SEARCH_VECTOR_EXPR}) STORED"
|
||||||
)
|
)
|
||||||
op.execute(
|
op.execute("CREATE INDEX ix_videos_search_vector ON videos USING gin (search_vector)")
|
||||||
"CREATE INDEX ix_videos_search_vector ON videos USING gin (search_vector)"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ un-subscribed channels):
|
|||||||
access to its ephemeral videos and drives the grace-period purge of un-kept channels.
|
access to its ephemeral videos and drives the grace-period purge of un-kept channels.
|
||||||
All new flags default false / NULL, so every existing row stays organic.
|
All new flags default false / NULL, so every existing row stays organic.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -30,9 +31,7 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column("channels", sa.Column("total_view_count", sa.BigInteger(), nullable=True))
|
op.add_column("channels", sa.Column("total_view_count", sa.BigInteger(), nullable=True))
|
||||||
op.add_column(
|
op.add_column("channels", sa.Column("published_at", sa.DateTime(timezone=True), nullable=True))
|
||||||
"channels", sa.Column("published_at", sa.DateTime(timezone=True), nullable=True)
|
|
||||||
)
|
|
||||||
op.add_column("channels", sa.Column("banner_url", sa.String(length=1024), nullable=True))
|
op.add_column("channels", sa.Column("banner_url", sa.String(length=1024), nullable=True))
|
||||||
op.add_column("channels", sa.Column("external_links", sa.JSON(), nullable=True))
|
op.add_column("channels", sa.Column("external_links", sa.JSON(), nullable=True))
|
||||||
op.add_column(
|
op.add_column(
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds the `blocked_channels` table: a per-user blocklist. Videos from a blocked c
|
|||||||
never shown in that user's live YouTube search (and not ingested), and are hidden from their
|
never shown in that user's live YouTube search (and not ingested), and are hidden from their
|
||||||
feed / explore views. Per-user, so one user's block doesn't affect anyone else.
|
feed / explore views. Per-user, so one user's block doesn't affect anyone else.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds the `saved_views` table: per-user named snapshots of the feed's filter/sort
|
|||||||
row stores the serialized FeedFilters (JSON), a sidebar `position` for drag-reorder, and an
|
row stores the serialized FeedFilters (JSON), a sidebar `position` for drag-reorder, and an
|
||||||
`is_default` flag (at most one per user, enforced by the API) for the view applied on load.
|
`is_default` flag (at most one per user, enforced by the API) for the view applied on load.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -26,9 +27,7 @@ def upgrade() -> None:
|
|||||||
sa.Column("user_id", sa.Integer(), nullable=False),
|
sa.Column("user_id", sa.Integer(), nullable=False),
|
||||||
sa.Column("name", sa.String(length=80), nullable=False),
|
sa.Column("name", sa.String(length=80), nullable=False),
|
||||||
sa.Column("filters", sa.JSON(), nullable=False),
|
sa.Column("filters", sa.JSON(), nullable=False),
|
||||||
sa.Column(
|
sa.Column("position", sa.Integer(), server_default=sa.text("0"), nullable=False),
|
||||||
"position", sa.Integer(), server_default=sa.text("0"), nullable=False
|
|
||||||
),
|
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"is_default",
|
"is_default",
|
||||||
sa.Boolean(),
|
sa.Boolean(),
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ FOR UPDATE SKIP LOCKED, live progress written back for polling); `download_profi
|
|||||||
reusable format presets (builtins seeded in the next migration); `download_shares` grant a file
|
reusable format presets (builtins seeded in the next migration); `download_shares` grant a file
|
||||||
to another user; `download_quotas` hold admin-set per-user limits (default via sysconfig).
|
to another user; `download_quotas` hold admin-set per-user limits (default via sysconfig).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -47,9 +48,16 @@ def upgrade() -> None:
|
|||||||
sa.Column("ref_count", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("ref_count", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("last_access_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("last_access_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"),
|
sa.UniqueConstraint(
|
||||||
|
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_media_assets_status", "media_assets", ["status"])
|
op.create_index("ix_media_assets_status", "media_assets", ["status"])
|
||||||
|
|
||||||
@@ -61,7 +69,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("spec", sa.JSON(), nullable=False),
|
sa.Column("spec", sa.JSON(), nullable=False),
|
||||||
sa.Column("is_builtin", sa.Boolean(), server_default="false", nullable=False),
|
sa.Column("is_builtin", sa.Boolean(), server_default="false", nullable=False),
|
||||||
sa.Column("sort_order", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("sort_order", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
)
|
)
|
||||||
@@ -84,8 +97,18 @@ def upgrade() -> None:
|
|||||||
sa.Column("phase", sa.String(length=32), nullable=True),
|
sa.Column("phase", sa.String(length=32), nullable=True),
|
||||||
sa.Column("error", sa.Text(), nullable=True),
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
sa.Column("queue_pos", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("queue_pos", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["asset_id"], ["media_assets.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["asset_id"], ["media_assets.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["profile_id"], ["download_profiles.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["profile_id"], ["download_profiles.id"], ondelete="SET NULL"),
|
||||||
@@ -102,7 +125,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("job_id", sa.Integer(), nullable=False),
|
sa.Column("job_id", sa.Integer(), nullable=False),
|
||||||
sa.Column("from_user_id", sa.Integer(), nullable=False),
|
sa.Column("from_user_id", sa.Integer(), nullable=False),
|
||||||
sa.Column("to_user_id", sa.Integer(), nullable=False),
|
sa.Column("to_user_id", sa.Integer(), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["from_user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["from_user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["to_user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["to_user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
@@ -120,7 +148,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("max_concurrent", sa.Integer(), nullable=False),
|
sa.Column("max_concurrent", sa.Integer(), nullable=False),
|
||||||
sa.Column("max_jobs", sa.Integer(), nullable=False),
|
sa.Column("max_jobs", sa.Integer(), nullable=False),
|
||||||
sa.Column("unlimited", sa.Boolean(), server_default="false", nullable=False),
|
sa.Column("unlimited", sa.Boolean(), server_default="false", nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("user_id"),
|
sa.PrimaryKeyConstraint("user_id"),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ definition consumed by app.downloads.formats: mode ("av" audio+video / "v" video
|
|||||||
the embed_*/sponsorblock postprocessor flags. Names are English (the UI default); the frontend
|
the embed_*/sponsorblock postprocessor flags. Names are English (the UI default); the frontend
|
||||||
localizes builtin labels by a stable key derived from the name.
|
localizes builtin labels by a stable key derived from the name.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ Create Date: 2026-07-03
|
|||||||
The download GC warns each holder once, a grace window before an asset's TTL expiry; this flag
|
The download GC warns each holder once, a grace window before an asset's TTL expiry; this flag
|
||||||
records that so subsequent GC runs don't re-notify until the asset is deleted.
|
records that so subsequent GC runs don't re-notify until the asset is deleted.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ display-friendly form (emoji stripped, ALL-CAPS de-shouted, trailing SEO hashtag
|
|||||||
see app.titles). Reversible: original_title preserves the source, and title can be re-derived.
|
see app.titles). Reversible: original_title preserves the source, and title can be re-derived.
|
||||||
The generated search_vector column regenerates automatically as each title is updated.
|
The generated search_vector column regenerates automatically as each title is updated.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -45,5 +46,7 @@ def upgrade() -> None:
|
|||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
conn = op.get_bind()
|
conn = op.get_bind()
|
||||||
conn.execute(sa.text("UPDATE videos SET title = original_title WHERE original_title IS NOT NULL"))
|
conn.execute(
|
||||||
|
sa.text("UPDATE videos SET title = original_title WHERE original_title IS NOT NULL")
|
||||||
|
)
|
||||||
op.drop_column("videos", "original_title")
|
op.drop_column("videos", "original_title")
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ rows so both fresh installs (0037 then this) and existing DBs converge on the sa
|
|||||||
jobs are unaffected (they snapshot their spec; profile_id just SET NULLs if it pointed at a
|
jobs are unaffected (they snapshot their spec; profile_id just SET NULLs if it pointed at a
|
||||||
replaced row).
|
replaced row).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ still hangs off a `media_assets` row (with `source_kind='edit'` and a per-user c
|
|||||||
existing file-serve / ref-count / GC / quota machinery is reused unchanged. `source_job_id` /
|
existing file-serve / ref-count / GC / quota machinery is reused unchanged. `source_job_id` /
|
||||||
`source_asset_id` point at the parent download; `edit_spec` is the trim/crop recipe.
|
`source_asset_id` point at the parent download; `edit_spec` is the trim/crop recipe.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -24,16 +25,10 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column(
|
op.add_column(
|
||||||
"download_jobs",
|
"download_jobs",
|
||||||
sa.Column(
|
sa.Column("job_kind", sa.String(length=16), server_default="download", nullable=False),
|
||||||
"job_kind", sa.String(length=16), server_default="download", nullable=False
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"download_jobs", sa.Column("source_job_id", sa.Integer(), nullable=True)
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"download_jobs", sa.Column("source_asset_id", sa.Integer(), nullable=True)
|
|
||||||
)
|
)
|
||||||
|
op.add_column("download_jobs", sa.Column("source_job_id", sa.Integer(), nullable=True))
|
||||||
|
op.add_column("download_jobs", sa.Column("source_asset_id", sa.Integer(), nullable=True))
|
||||||
op.add_column("download_jobs", sa.Column("edit_spec", sa.JSON(), nullable=True))
|
op.add_column("download_jobs", sa.Column("edit_spec", sa.JSON(), nullable=True))
|
||||||
op.create_foreign_key(
|
op.create_foreign_key(
|
||||||
"fk_download_jobs_source_job",
|
"fk_download_jobs_source_job",
|
||||||
@@ -54,12 +49,8 @@ def upgrade() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.drop_constraint(
|
op.drop_constraint("fk_download_jobs_source_asset", "download_jobs", type_="foreignkey")
|
||||||
"fk_download_jobs_source_asset", "download_jobs", type_="foreignkey"
|
op.drop_constraint("fk_download_jobs_source_job", "download_jobs", type_="foreignkey")
|
||||||
)
|
|
||||||
op.drop_constraint(
|
|
||||||
"fk_download_jobs_source_job", "download_jobs", type_="foreignkey"
|
|
||||||
)
|
|
||||||
op.drop_column("download_jobs", "edit_spec")
|
op.drop_column("download_jobs", "edit_spec")
|
||||||
op.drop_column("download_jobs", "source_asset_id")
|
op.drop_column("download_jobs", "source_asset_id")
|
||||||
op.drop_column("download_jobs", "source_job_id")
|
op.drop_column("download_jobs", "source_job_id")
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ can watch the file on the public `/watch/{token}` player page — no account nee
|
|||||||
per-link controls: an expiry, an "allow download" toggle, and an (argon2-hashed) password. Revoke
|
per-link controls: an expiry, an "allow download" toggle, and an (argon2-hashed) password. Revoke
|
||||||
= delete the row. Distinct from `download_shares` (an ACL grant to another *registered* user).
|
= delete the row. Distinct from `download_shares` (an ACL grant to another *registered* user).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -31,7 +32,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("password_hash", sa.Text(), nullable=True),
|
sa.Column("password_hash", sa.Text(), nullable=True),
|
||||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("view_count", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("view_count", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["created_by"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["created_by"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Stores yt-dlp's canonical `webpage_url` for a download's source, so the Download
|
|||||||
clean "downloaded from" reference (copyable + openable). Nullable — filled by the worker on
|
clean "downloaded from" reference (copyable + openable). Nullable — filled by the worker on
|
||||||
extraction; existing/queued rows fall back to a URL derived from source_kind+source_ref.
|
extraction; existing/queued rows fall back to a URL derived from source_kind+source_ref.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ Plex server (playback is from the local physical file; Plex is metadata only). T
|
|||||||
The generated `search_vector` columns + their GIN indexes are added via raw SQL (same approach
|
The generated `search_vector` columns + their GIN indexes are added via raw SQL (same approach
|
||||||
as migration 0032), reusing the `unaccent_simple` text-search config from migration 0031.
|
as migration 0032), reusing the `unaccent_simple` text-search config from migration 0031.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -52,8 +53,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("kind", sa.String(length=16), nullable=False),
|
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||||
sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False),
|
sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False),
|
||||||
sa.Column("synced_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("synced_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_libraries_plex_key", "plex_libraries", ["plex_key"], unique=True)
|
op.create_index("ix_plex_libraries_plex_key", "plex_libraries", ["plex_key"], unique=True)
|
||||||
|
|
||||||
@@ -61,7 +66,12 @@ def upgrade() -> None:
|
|||||||
"plex_shows",
|
"plex_shows",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
||||||
sa.Column("library_id", sa.Integer(), sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"library_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("title", sa.String(length=512), nullable=False),
|
sa.Column("title", sa.String(length=512), nullable=False),
|
||||||
sa.Column("summary", sa.Text(), nullable=True),
|
sa.Column("summary", sa.Text(), nullable=True),
|
||||||
sa.Column("year", sa.Integer(), nullable=True),
|
sa.Column("year", sa.Integer(), nullable=True),
|
||||||
@@ -69,8 +79,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("art_key", sa.String(length=512), nullable=True),
|
sa.Column("art_key", sa.String(length=512), nullable=True),
|
||||||
sa.Column("child_count", sa.Integer(), nullable=True),
|
sa.Column("child_count", sa.Integer(), nullable=True),
|
||||||
sa.Column("added_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("added_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_shows_rating_key", "plex_shows", ["rating_key"], unique=True)
|
op.create_index("ix_plex_shows_rating_key", "plex_shows", ["rating_key"], unique=True)
|
||||||
op.create_index("ix_plex_shows_library_id", "plex_shows", ["library_id"])
|
op.create_index("ix_plex_shows_library_id", "plex_shows", ["library_id"])
|
||||||
@@ -81,12 +95,21 @@ def upgrade() -> None:
|
|||||||
"plex_seasons",
|
"plex_seasons",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
||||||
sa.Column("show_id", sa.Integer(), sa.ForeignKey("plex_shows.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"show_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_shows.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("title", sa.String(length=255), nullable=True),
|
sa.Column("title", sa.String(length=255), nullable=True),
|
||||||
sa.Column("season_number", sa.Integer(), nullable=True),
|
sa.Column("season_number", sa.Integer(), nullable=True),
|
||||||
sa.Column("thumb_key", sa.String(length=512), nullable=True),
|
sa.Column("thumb_key", sa.String(length=512), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_seasons_rating_key", "plex_seasons", ["rating_key"], unique=True)
|
op.create_index("ix_plex_seasons_rating_key", "plex_seasons", ["rating_key"], unique=True)
|
||||||
op.create_index("ix_plex_seasons_show_id", "plex_seasons", ["show_id"])
|
op.create_index("ix_plex_seasons_show_id", "plex_seasons", ["show_id"])
|
||||||
@@ -96,10 +119,25 @@ def upgrade() -> None:
|
|||||||
"plex_items",
|
"plex_items",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
||||||
sa.Column("library_id", sa.Integer(), sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"library_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("kind", sa.String(length=16), nullable=False),
|
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||||
sa.Column("show_id", sa.Integer(), sa.ForeignKey("plex_shows.id", ondelete="CASCADE"), nullable=True),
|
sa.Column(
|
||||||
sa.Column("season_id", sa.Integer(), sa.ForeignKey("plex_seasons.id", ondelete="CASCADE"), nullable=True),
|
"show_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_shows.id", ondelete="CASCADE"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"season_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_seasons.id", ondelete="CASCADE"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
sa.Column("season_number", sa.Integer(), nullable=True),
|
sa.Column("season_number", sa.Integer(), nullable=True),
|
||||||
sa.Column("episode_number", sa.Integer(), nullable=True),
|
sa.Column("episode_number", sa.Integer(), nullable=True),
|
||||||
sa.Column("title", sa.String(length=512), nullable=False),
|
sa.Column("title", sa.String(length=512), nullable=False),
|
||||||
@@ -119,8 +157,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("probed_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("probed_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("markers", sa.JSON(), nullable=True),
|
sa.Column("markers", sa.JSON(), nullable=True),
|
||||||
sa.Column("added_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("added_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_items_rating_key", "plex_items", ["rating_key"], unique=True)
|
op.create_index("ix_plex_items_rating_key", "plex_items", ["rating_key"], unique=True)
|
||||||
op.create_index("ix_plex_items_library_id", "plex_items", ["library_id"])
|
op.create_index("ix_plex_items_library_id", "plex_items", ["library_id"])
|
||||||
@@ -137,14 +179,23 @@ def upgrade() -> None:
|
|||||||
op.create_table(
|
op.create_table(
|
||||||
"plex_states",
|
"plex_states",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
sa.Column("item_id", sa.Integer(), sa.ForeignKey("plex_items.id", ondelete="CASCADE"), nullable=False),
|
"user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"item_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_items.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("status", sa.String(length=12), server_default="new", nullable=False),
|
sa.Column("status", sa.String(length=12), server_default="new", nullable=False),
|
||||||
sa.Column("position_seconds", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("position_seconds", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("watched_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("watched_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("progress_updated_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("progress_updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("synced_to_plex", sa.Boolean(), server_default="false", nullable=False),
|
sa.Column("synced_to_plex", sa.Boolean(), server_default="false", nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
sa.UniqueConstraint("user_id", "item_id", name="uq_plex_user_item"),
|
sa.UniqueConstraint("user_id", "item_id", name="uq_plex_user_item"),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_states_user_id", "plex_states", ["user_id"])
|
op.create_index("ix_plex_states_user_id", "plex_states", ["user_id"])
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ Plex section listing (no per-item API calls): rating (audienceRating ~ IMDb), co
|
|||||||
release date, and GIN-indexed genre / director / cast arrays. These also double as the content
|
release date, and GIN-indexed genre / director / cast arrays. These also double as the content
|
||||||
signals a future watch-habit recommender would use.
|
signals a future watch-habit recommender would use.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -33,9 +34,13 @@ def upgrade() -> None:
|
|||||||
op.create_index("ix_plex_items_rating", "plex_items", ["rating"])
|
op.create_index("ix_plex_items_rating", "plex_items", ["rating"])
|
||||||
op.create_index("ix_plex_items_content_rating", "plex_items", ["content_rating"])
|
op.create_index("ix_plex_items_content_rating", "plex_items", ["content_rating"])
|
||||||
op.create_index("ix_plex_items_studio", "plex_items", ["studio"])
|
op.create_index("ix_plex_items_studio", "plex_items", ["studio"])
|
||||||
op.create_index("ix_plex_items_originally_available_at", "plex_items", ["originally_available_at"])
|
op.create_index(
|
||||||
|
"ix_plex_items_originally_available_at", "plex_items", ["originally_available_at"]
|
||||||
|
)
|
||||||
op.create_index("ix_plex_items_genres_gin", "plex_items", ["genres"], postgresql_using="gin")
|
op.create_index("ix_plex_items_genres_gin", "plex_items", ["genres"], postgresql_using="gin")
|
||||||
op.create_index("ix_plex_items_directors_gin", "plex_items", ["directors"], postgresql_using="gin")
|
op.create_index(
|
||||||
|
"ix_plex_items_directors_gin", "plex_items", ["directors"], postgresql_using="gin"
|
||||||
|
)
|
||||||
op.create_index("ix_plex_items_cast_gin", "plex_items", ["cast_names"], postgresql_using="gin")
|
op.create_index("ix_plex_items_cast_gin", "plex_items", ["cast_names"], postgresql_using="gin")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ finds titles by an actor/director name and ranks those above films that merely m
|
|||||||
their synopsis. A generated column's expression can't be altered in place, so we drop + recreate it
|
their synopsis. A generated column's expression can't be altered in place, so we drop + recreate it
|
||||||
(and its GIN index). `people_text` is populated on the next Plex sync.
|
(and its GIN index). `people_text` is populated on the next Plex sync.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ A `plex_collections` table holds each collection's card metadata; membership is
|
|||||||
combinable filter and a title's "collection strip" is a pure local lookup. `editable` marks the
|
combinable filter and a title's "collection strip" is a pure local lookup. `editable` marks the
|
||||||
(curated) collections we may later write back to Plex; everything is read-only by default.
|
(curated) collections we may later write back to Plex; everything is read-only by default.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -27,7 +28,12 @@ def upgrade() -> None:
|
|||||||
"plex_collections",
|
"plex_collections",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
||||||
sa.Column("library_id", sa.Integer(), sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"library_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("title", sa.String(length=512), nullable=False),
|
sa.Column("title", sa.String(length=512), nullable=False),
|
||||||
sa.Column("summary", sa.Text(), nullable=True),
|
sa.Column("summary", sa.Text(), nullable=True),
|
||||||
sa.Column("thumb_key", sa.String(length=512), nullable=True),
|
sa.Column("thumb_key", sa.String(length=512), nullable=True),
|
||||||
@@ -35,16 +41,32 @@ def upgrade() -> None:
|
|||||||
sa.Column("smart", sa.Boolean(), nullable=False, server_default="false"),
|
sa.Column("smart", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
sa.Column("editable", sa.Boolean(), nullable=False, server_default="false"),
|
sa.Column("editable", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
sa.Column("synced_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("synced_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_plex_collections_rating_key", "plex_collections", ["rating_key"], unique=True
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_collections_rating_key", "plex_collections", ["rating_key"], unique=True)
|
|
||||||
op.create_index("ix_plex_collections_library_id", "plex_collections", ["library_id"])
|
op.create_index("ix_plex_collections_library_id", "plex_collections", ["library_id"])
|
||||||
|
|
||||||
op.add_column("plex_items", sa.Column("collection_keys", postgresql.JSONB(), nullable=True))
|
op.add_column("plex_items", sa.Column("collection_keys", postgresql.JSONB(), nullable=True))
|
||||||
op.create_index("ix_plex_items_collections_gin", "plex_items", ["collection_keys"], postgresql_using="gin")
|
op.create_index(
|
||||||
|
"ix_plex_items_collections_gin", "plex_items", ["collection_keys"], postgresql_using="gin"
|
||||||
|
)
|
||||||
op.add_column("plex_shows", sa.Column("collection_keys", postgresql.JSONB(), nullable=True))
|
op.add_column("plex_shows", sa.Column("collection_keys", postgresql.JSONB(), nullable=True))
|
||||||
op.create_index("ix_plex_shows_collections_gin", "plex_shows", ["collection_keys"], postgresql_using="gin")
|
op.create_index(
|
||||||
|
"ix_plex_shows_collections_gin", "plex_shows", ["collection_keys"], postgresql_using="gin"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ Plex account (like plex_states) — the "your own lists" counterpart to the shar
|
|||||||
collections. `plex_playlists` is owned by a user; `plex_playlist_items` holds the ordered entries
|
collections. `plex_playlists` is owned by a user; `plex_playlist_items` holds the ordered entries
|
||||||
(`position`). `plex_rating_key` is reserved for an optional later Plex-direction sync.
|
(`position`). `plex_rating_key` is reserved for an optional later Plex-direction sync.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -24,11 +25,23 @@ def upgrade() -> None:
|
|||||||
op.create_table(
|
op.create_table(
|
||||||
"plex_playlists",
|
"plex_playlists",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||||
|
),
|
||||||
sa.Column("title", sa.String(length=512), nullable=False),
|
sa.Column("title", sa.String(length=512), nullable=False),
|
||||||
sa.Column("plex_rating_key", sa.String(length=32), nullable=True),
|
sa.Column("plex_rating_key", sa.String(length=32), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_playlists_user_id", "plex_playlists", ["user_id"])
|
op.create_index("ix_plex_playlists_user_id", "plex_playlists", ["user_id"])
|
||||||
|
|
||||||
@@ -36,12 +49,16 @@ def upgrade() -> None:
|
|||||||
"plex_playlist_items",
|
"plex_playlist_items",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"playlist_id", sa.Integer(),
|
"playlist_id",
|
||||||
sa.ForeignKey("plex_playlists.id", ondelete="CASCADE"), nullable=False,
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_playlists.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
),
|
),
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"item_id", sa.Integer(),
|
"item_id",
|
||||||
sa.ForeignKey("plex_items.id", ondelete="CASCADE"), nullable=False,
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_items.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
),
|
),
|
||||||
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
||||||
sa.UniqueConstraint("playlist_id", "item_id", name="uq_plex_playlist_item"),
|
sa.UniqueConstraint("playlist_id", "item_id", name="uq_plex_playlist_item"),
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ and captures yt-dlp's canonical channel/uploader URL so an auto-filled channel b
|
|||||||
overrides (like display_name — display metadata only, never touches the on-disk file).
|
overrides (like display_name — display metadata only, never touches the on-disk file).
|
||||||
- download_jobs.extra_links: optional list of extra reference URLs shown on the card + watch page.
|
- download_jobs.extra_links: optional list of extra reference URLs shown on the card + watch page.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -26,7 +27,9 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column("media_assets", sa.Column("uploader_url", sa.Text(), nullable=True))
|
op.add_column("media_assets", sa.Column("uploader_url", sa.Text(), nullable=True))
|
||||||
op.add_column("download_jobs", sa.Column("display_uploader", sa.String(length=255), nullable=True))
|
op.add_column(
|
||||||
|
"download_jobs", sa.Column("display_uploader", sa.String(length=255), nullable=True)
|
||||||
|
)
|
||||||
op.add_column("download_jobs", sa.Column("display_uploader_url", sa.Text(), nullable=True))
|
op.add_column("download_jobs", sa.Column("display_uploader_url", sa.Text(), nullable=True))
|
||||||
op.add_column("download_jobs", sa.Column("extra_links", sa.JSON(), nullable=True))
|
op.add_column("download_jobs", sa.Column("extra_links", sa.JSON(), nullable=True))
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ download shows a blank image box. The worker now extracts a representative frame
|
|||||||
saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
|
saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
|
||||||
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
|
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ state with a Plex account. The owner MVP uses the server admin token (uses_admin
|
|||||||
token); a personal token column is reserved for later friend-account linking (PIN OAuth). See the
|
token); a personal token column is reserved for later friend-account linking (PIN OAuth). See the
|
||||||
PlexLink model docstring.
|
PlexLink model docstring.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ blob folded into the generated `search_vector` (weight B). This lets the TV grid
|
|||||||
sorted like the movie grid. A generated column's expression can't be altered in place, so the
|
sorted like the movie grid. A generated column's expression can't be altered in place, so the
|
||||||
search_vector is dropped + recreated. All new columns are populated on the next Plex sync.
|
search_vector is dropped + recreated. All new columns are populated on the next Plex sync.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -57,9 +58,13 @@ def upgrade() -> None:
|
|||||||
op.create_index("ix_plex_shows_rating", "plex_shows", ["rating"])
|
op.create_index("ix_plex_shows_rating", "plex_shows", ["rating"])
|
||||||
op.create_index("ix_plex_shows_content_rating", "plex_shows", ["content_rating"])
|
op.create_index("ix_plex_shows_content_rating", "plex_shows", ["content_rating"])
|
||||||
op.create_index("ix_plex_shows_studio", "plex_shows", ["studio"])
|
op.create_index("ix_plex_shows_studio", "plex_shows", ["studio"])
|
||||||
op.create_index("ix_plex_shows_originally_available_at", "plex_shows", ["originally_available_at"])
|
op.create_index(
|
||||||
|
"ix_plex_shows_originally_available_at", "plex_shows", ["originally_available_at"]
|
||||||
|
)
|
||||||
op.create_index("ix_plex_shows_genres_gin", "plex_shows", ["genres"], postgresql_using="gin")
|
op.create_index("ix_plex_shows_genres_gin", "plex_shows", ["genres"], postgresql_using="gin")
|
||||||
op.create_index("ix_plex_shows_directors_gin", "plex_shows", ["directors"], postgresql_using="gin")
|
op.create_index(
|
||||||
|
"ix_plex_shows_directors_gin", "plex_shows", ["directors"], postgresql_using="gin"
|
||||||
|
)
|
||||||
op.create_index("ix_plex_shows_cast_gin", "plex_shows", ["cast_names"], postgresql_using="gin")
|
op.create_index("ix_plex_shows_cast_gin", "plex_shows", ["cast_names"], postgresql_using="gin")
|
||||||
# NB: ix_plex_shows_collections_gin already exists (created with collections in 0047).
|
# NB: ix_plex_shows_collections_gin already exists (created with collections in 0047).
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ current_user rejects any cookie whose epoch is stale. Bumped on password reset,
|
|||||||
or "log out everywhere" — so a stolen/copied client-side cookie dies on those events instead of
|
or "log out everywhere" — so a stolen/copied client-side cookie dies on those events instead of
|
||||||
staying valid until it expires. Existing rows default to 0 (matching a fresh cookie's absent/0 epoch).
|
staying valid until it expires. Existing rows default to 0 (matching a fresh cookie's absent/0 epoch).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0054_audit_log
|
|||||||
Revises: 0053_user_session_epoch
|
Revises: 0053_user_session_epoch
|
||||||
Create Date: 2026-07-12
|
Create Date: 2026-07-12
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0055_channel_keywords
|
|||||||
Revises: 0054_audit_log
|
Revises: 0054_audit_log
|
||||||
Create Date: 2026-07-12
|
Create Date: 2026-07-12
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0056_plex_people
|
|||||||
Revises: 0055_channel_keywords
|
Revises: 0055_channel_keywords
|
||||||
Create Date: 2026-07-19
|
Create Date: 2026-07-19
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -23,8 +24,12 @@ def upgrade() -> None:
|
|||||||
# A representative proxied headshot URL (or NULL when none is available). Captured
|
# A representative proxied headshot URL (or NULL when none is available). Captured
|
||||||
# opportunistically from an item's rich metadata (info-page view + the rich re-sync).
|
# opportunistically from an item's rich metadata (info-page view + the rich re-sync).
|
||||||
sa.Column("photo_thumb", sa.Text(), nullable=True),
|
sa.Column("photo_thumb", sa.Text(), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_people_name", "plex_people", ["name"], unique=True)
|
op.create_index("ix_plex_people_name", "plex_people", ["name"], unique=True)
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0057_plex_ppl_enriched
|
|||||||
Revises: 0056_plex_people
|
Revises: 0056_plex_people
|
||||||
Create Date: 2026-07-19
|
Create Date: 2026-07-19
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -17,7 +18,9 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
for tbl in ("plex_items", "plex_shows"):
|
for tbl in ("plex_items", "plex_shows"):
|
||||||
op.add_column(tbl, sa.Column("people_enriched_at", sa.DateTime(timezone=True), nullable=True))
|
op.add_column(
|
||||||
|
tbl, sa.Column("people_enriched_at", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ Best still one click away. Re-seeds the builtin rows so fresh installs and exist
|
|||||||
(same mechanism as 0040). Existing jobs are unaffected (they snapshot their spec; a profile_id that
|
(same mechanism as 0040). Existing jobs are unaffected (they snapshot their spec; a profile_id that
|
||||||
pointed at a replaced row just SET NULLs).
|
pointed at a replaced row just SET NULLs).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ did NOT invalidate outstanding grants — they lived out their 6h TTL. Fold a pe
|
|||||||
into the signed grant and bump it on every password change, so a rotation revokes old grants at once.
|
into the signed grant and bump it on every password change, so a rotation revokes old grants at once.
|
||||||
Small table (one row per share link), so this is a quick add-column with a default.
|
Small table (one row per share link), so this is a quick add-column with a default.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ This one-time data fixup rewrites those scalars to SQL NULL so the columns are u
|
|||||||
forward-only: a SQL NULL is indistinguishable from the old JSON `null` to every reader (the guards
|
forward-only: a SQL NULL is indistinguishable from the old JSON `null` to every reader (the guards
|
||||||
treat both as "not an array"), so there is nothing meaningful to downgrade. No schema change.
|
treat both as "not an array"), so there is nothing meaningful to downgrade. No schema change.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -25,9 +26,7 @@ _COLS = ("genres", "directors", "cast_names", "collection_keys")
|
|||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
for table in _TABLES:
|
for table in _TABLES:
|
||||||
for col in _COLS:
|
for col in _COLS:
|
||||||
op.execute(
|
op.execute(f"UPDATE {table} SET {col} = NULL WHERE jsonb_typeof({col}) = 'null'")
|
||||||
f"UPDATE {table} SET {col} = NULL WHERE jsonb_typeof({col}) = 'null'"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ small category code (`unavailable` / `login_required` / `geo_blocked` / `format_
|
|||||||
a retry makes sense, keeping the raw text as a technical-detail disclosure. Nullable; only set on the
|
a retry makes sense, keeping the raw text as a technical-detail disclosure. Nullable; only set on the
|
||||||
error path. Forward-only — an existing errored row simply has NULL until its next failure/retry.
|
error path. Forward-only — an existing errored row simply has NULL until its next failure/retry.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ Redundant index -> the composite that already covers it:
|
|||||||
ix_plex_states_user_id <- uq_plex_user_item (user_id, item_id)
|
ix_plex_states_user_id <- uq_plex_user_item (user_id, item_id)
|
||||||
ix_plex_playlist_items_playlist_id <- uq_plex_playlist_item (playlist_id, item_id)
|
ix_plex_playlist_items_playlist_id <- uq_plex_playlist_item (playlist_id, item_id)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ OVERLAY library is the Hungarian-metadata twin of another section: it is never b
|
|||||||
its titles' HU fields are written onto the matched EN rows (paired by `guid`, see 0064). Cheap:
|
its titles' HU fields are written onto the matched EN rows (paired by `guid`, see 0064). Cheap:
|
||||||
metadata-only, no table rewrite.
|
metadata-only, no table rewrite.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ Run BEFORE the first HU overlay sync: while the *_hu columns are all NULL the ge
|
|||||||
is trivial. ADD only (no drop/rebuild of the EN search_vector). NOTE: adding a STORED generated
|
is trivial. ADD only (no drop/rebuild of the EN search_vector). NOTE: adding a STORED generated
|
||||||
column + its GIN takes an ACCESS EXCLUSIVE lock and rewrites the table — brief, but flag it on prod.
|
column + its GIN takes an ACCESS EXCLUSIVE lock and rewrites the table — brief, but flag it on prod.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ so the data step below resets `people_enriched_at` to NULL on movie/show rows: t
|
|||||||
each once (60/run, self-throttled) and stamps the new columns. ADD-only, all nullable → no table
|
each once (60/run, self-throttled) and stamps the new columns. ADD-only, all nullable → no table
|
||||||
rewrite, no generated column, no ACCESS EXCLUSIVE lock (unlike 0064).
|
rewrite, no generated column, no ACCESS EXCLUSIVE lock (unlike 0064).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ schema allowed NULL — a model/schema drift the new migration drift-test (E1 S2
|
|||||||
server_default has filled every row since day one, so no NULLs exist; enforcing NOT NULL just makes the
|
server_default has filled every row since day one, so no NULLs exist; enforcing NOT NULL just makes the
|
||||||
schema match the contract the model already promises. Pure DDL, reversible.
|
schema match the contract the model already promises. Pure DDL, reversible.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
"""feed sort indexes: published_at DESC NULLS LAST composite(s) for the default feed (PERF-1)
|
||||||
|
|
||||||
|
The feed's default `newest` sort is `published_at DESC NULLS LAST, id ASC` (feedquery.order_by), but the
|
||||||
|
only published_at index was a plain ASC btree (`ix_videos_published_at`). A btree read backwards satisfies
|
||||||
|
`DESC NULLS FIRST`, not the `DESC NULLS LAST` the query asks for, so the planner could not use it for
|
||||||
|
ordering — every feed page fell back to a full `videos` scan + top-N sort. Measured on a prod-sized reseed
|
||||||
|
(257k videos): the default page went from ~44.8k shared buffers (dev planner) / ~169k (prod planner, the
|
||||||
|
per-channel nested loop that is the likely "prod feed is slow" root cause) down to ~624 buffers with the
|
||||||
|
matching index.
|
||||||
|
|
||||||
|
This adds two indexes in the exact feed order:
|
||||||
|
- ix_videos_published_at_desc (published_at DESC NULLS LAST, id ASC) — the global feed / Library sort.
|
||||||
|
- ix_videos_channel_pub_desc (channel_id, published_at DESC NULLS LAST, id ASC) — the per-channel feed
|
||||||
|
(channel page, channel_ids filter). It leads with channel_id, so it is a strict superset of the old
|
||||||
|
single-column `ix_videos_channel_id` (FK cascade, sync upsert, autotag, channel-detail all still ride
|
||||||
|
it) — that redundant index is dropped.
|
||||||
|
|
||||||
|
The plain ASC `ix_videos_published_at` stays: it already serves the `oldest` sort (ASC NULLS LAST = btree
|
||||||
|
default) as an Index Scan.
|
||||||
|
|
||||||
|
CONCURRENTLY (in alembic's autocommit_block, since it can't run inside a transaction) so a live prod builds
|
||||||
|
these without blocking reads/writes; downtime ~0. IF (NOT) EXISTS keeps the drift test's
|
||||||
|
downgrade->re-upgrade round-trip idempotent.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0067_feed_sort_indexes"
|
||||||
|
down_revision: Union[str, None] = "0066_notif_created_at_notnull"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.get_context().autocommit_block():
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_published_at_desc "
|
||||||
|
"ON videos (published_at DESC NULLS LAST, id ASC)"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_channel_pub_desc "
|
||||||
|
"ON videos (channel_id, published_at DESC NULLS LAST, id ASC)"
|
||||||
|
)
|
||||||
|
# Redundant now — ix_videos_channel_pub_desc leads with channel_id (a strict superset).
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_channel_id")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.get_context().autocommit_block():
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_channel_id ON videos (channel_id)"
|
||||||
|
)
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_channel_pub_desc")
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_published_at_desc")
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
"""videos.published_at NOT NULL DEFAULT now() (PERF-15)
|
||||||
|
|
||||||
|
Every video in practice carries a publish date (0 NULLs in the 257k-row prod-sized reseed), but the
|
||||||
|
column was nullable, which forced the feed's keyset predicate into a non-sargable
|
||||||
|
`... OR published_at IS NULL` disjunction — blocking the ix_videos_published_at_desc range so deep
|
||||||
|
`newest` pages scanned from the top of the index and filtered (51.8k buffers). Making it NOT NULL lets
|
||||||
|
the keyset drop that disjunction and take a `published_at <=` range bound (deep page -> ~0.5k buffers).
|
||||||
|
|
||||||
|
server_default now() is the safety net for the rare date-less ingest path (the enrich overwrites it with
|
||||||
|
the real date); the app-side ingest paths also fall back to now() so they never write an explicit NULL.
|
||||||
|
|
||||||
|
Note: `SET NOT NULL` validates with a brief ACCESS EXCLUSIVE lock + one table scan (it cannot run
|
||||||
|
CONCURRENTLY). On the prod row count (~235k) that is sub-second; still, it is a short write-blocking lock
|
||||||
|
rather than the ~0 of 0067's CONCURRENTLY index build.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0068_published_at_not_null"
|
||||||
|
down_revision: Union[str, None] = "0067_feed_sort_indexes"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# Backfill any stray NULLs first so SET NOT NULL can't fail (0 in prod, but be safe).
|
||||||
|
op.execute("UPDATE videos SET published_at = now() WHERE published_at IS NULL")
|
||||||
|
op.alter_column("videos", "published_at", nullable=False, server_default=sa.text("now()"))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.alter_column("videos", "published_at", nullable=True, server_default=None)
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
"""scheduler-job partial indexes + boolean-index cleanup (PERF-17/18/3)
|
||||||
|
|
||||||
|
Two scheduler jobs full-seq-scanned the 257k-row videos table every run:
|
||||||
|
- PERF-17: enrich_pending (`WHERE enriched_at IS NULL LIMIT N`, every few minutes) — 44.6k buffers /
|
||||||
|
run even when nothing is pending. A partial index over just the pending rows makes it a 1-buffer
|
||||||
|
(empty) index scan.
|
||||||
|
- PERF-18: maintenance revalidate (`WHERE unavailable_since IS NULL ORDER BY last_checked_at NULLS
|
||||||
|
FIRST LIMIT N`, ~300 pages/run) — 44.7k buffers/page; ix_videos_last_checked_at had idx_scan=0. A
|
||||||
|
partial index in that exact order serves it as an index scan (~36 buffers/page) and replaces the
|
||||||
|
unused plain one.
|
||||||
|
|
||||||
|
PERF-3: the videos table also carried plain btrees on low-cardinality boolean columns the planner never
|
||||||
|
picks (a `col IS false` majority predicate can't ride a btree). Drop the never-useful ones (is_short,
|
||||||
|
via_explore) and slim the rest to partial indexes over the small/shrinking subset the job queries target
|
||||||
|
(via_search=true, NOT shorts_probed, unavailable_since IS NOT NULL) — less write amplification on the
|
||||||
|
biggest table, same lookups. live_status is left alone (a separate finding).
|
||||||
|
|
||||||
|
All CONCURRENTLY (autocommit_block) — downtime ~0; IF (NOT) EXISTS keeps the drift test's
|
||||||
|
downgrade->re-upgrade idempotent.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0069_job_index_hygiene"
|
||||||
|
down_revision: Union[str, None] = "0068_published_at_not_null"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.get_context().autocommit_block():
|
||||||
|
# PERF-17: enrich-pending partial (empty when nothing is pending -> 1-buffer scan).
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_enrich_pending "
|
||||||
|
"ON videos (id) WHERE enriched_at IS NULL"
|
||||||
|
)
|
||||||
|
# PERF-18: maintenance revalidate order, as a partial in the exact WHERE + ORDER BY shape.
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_recheck_order "
|
||||||
|
"ON videos (last_checked_at ASC NULLS FIRST) WHERE unavailable_since IS NULL"
|
||||||
|
)
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_last_checked_at")
|
||||||
|
# PERF-3: drop the never-picked boolean btrees; slim the rest to partials over the target subset.
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_is_short")
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_via_explore")
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_via_search")
|
||||||
|
# Predicate MUST match the query's `.is_(True)`/`.is_(False)` form (SQL `IS TRUE`/`IS FALSE`):
|
||||||
|
# Postgres's partial-index prover does NOT match a `col IS TRUE` clause to a bare `col` predicate,
|
||||||
|
# so `WHERE via_search` / `WHERE NOT shorts_probed` would leave the queries seq-scanning.
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_via_search "
|
||||||
|
"ON videos (id) WHERE via_search IS TRUE"
|
||||||
|
)
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_shorts_probed")
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_shorts_probed "
|
||||||
|
"ON videos (id) WHERE shorts_probed IS FALSE"
|
||||||
|
)
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_unavailable_since")
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_unavailable_since "
|
||||||
|
"ON videos (unavailable_since) WHERE unavailable_since IS NOT NULL"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.get_context().autocommit_block():
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_enrich_pending")
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_recheck_order")
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_last_checked_at "
|
||||||
|
"ON videos (last_checked_at)"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_is_short ON videos (is_short)"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_via_explore ON videos (via_explore)"
|
||||||
|
)
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_via_search")
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_via_search ON videos (via_search)"
|
||||||
|
)
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_shorts_probed")
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_shorts_probed ON videos (shorts_probed)"
|
||||||
|
)
|
||||||
|
op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_unavailable_since")
|
||||||
|
op.execute(
|
||||||
|
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_videos_unavailable_since "
|
||||||
|
"ON videos (unavailable_since)"
|
||||||
|
)
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
"""Fold the maintenance re-validation batch from app_state into sysconfig (B-core-11)
|
||||||
|
|
||||||
|
The admin-tunable `maintenance_revalidate_batch` lived in its own `app_state` column with a
|
||||||
|
hand-rolled override-or-default reader in `app.state`, duplicating the sysconfig SPECS pattern. It's
|
||||||
|
now a sysconfig key (`system_config` store), so every admin knob lives in one place. This migration
|
||||||
|
carries any existing override into the sysconfig store, then drops the dead column. Reversible.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0070_fold_maintenance_batch"
|
||||||
|
down_revision: Union[str, None] = "0069_job_index_hygiene"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# Preserve a customized override: copy it into the sysconfig store before the column goes.
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO system_config (key, value, encrypted)
|
||||||
|
SELECT 'maintenance_revalidate_batch', maintenance_revalidate_batch::text, false
|
||||||
|
FROM app_state
|
||||||
|
WHERE id = 1 AND maintenance_revalidate_batch IS NOT NULL
|
||||||
|
ON CONFLICT (key) DO NOTHING
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.drop_column("app_state", "maintenance_revalidate_batch")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"app_state",
|
||||||
|
sa.Column("maintenance_revalidate_batch", sa.Integer(), nullable=True),
|
||||||
|
)
|
||||||
|
# Carry the value back onto the row, then drop the sysconfig override (symmetric restore).
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE app_state SET maintenance_revalidate_batch =
|
||||||
|
(SELECT value::int FROM system_config WHERE key = 'maintenance_revalidate_batch')
|
||||||
|
WHERE id = 1
|
||||||
|
AND EXISTS (SELECT 1 FROM system_config WHERE key = 'maintenance_revalidate_batch')
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.execute("DELETE FROM system_config WHERE key = 'maintenance_revalidate_batch'")
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
"""plex TMDB enrichment columns (E5 S2)
|
||||||
|
|
||||||
|
Additive TMDB-sourced fields on plex_items + plex_shows — keywords, full crew, budget/revenue, TMDb
|
||||||
|
community score, production companies/countries, recommendations, and a backdrop-path pool — plus a
|
||||||
|
`tmdb_id` cache and a `tmdb_enriched_at` watermark. Filled by the new TMDB enrichment job (matched by
|
||||||
|
the stored canonical `guid`), SERVED FROM THE ROW (never a live TMDB call). Distinct from the existing
|
||||||
|
Plex-sourced `imdb_rating`/`rating`, which are left untouched. Plus `plex_people.tmdb_person_id` (the S2
|
||||||
|
foundation for S6's person bios). ADD-only, all nullable → no table rewrite, no ACCESS EXCLUSIVE lock.
|
||||||
|
No watermark reset needed: `tmdb_enriched_at` starts NULL, so every eligible row is picked up once.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0071_plex_tmdb_fields"
|
||||||
|
down_revision: Union[str, None] = "0070_fold_maintenance_batch"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
_TABLES = ("plex_items", "plex_shows")
|
||||||
|
|
||||||
|
# (name, type) — same set on both movie/leaf rows and show rows. budget/revenue stay NULL for shows.
|
||||||
|
_COLS: tuple[tuple[str, sa.types.TypeEngine], ...] = (
|
||||||
|
("tmdb_id", sa.Integer()),
|
||||||
|
("tmdb_enriched_at", sa.DateTime(timezone=True)),
|
||||||
|
("tmdb_rating", sa.Float()),
|
||||||
|
("tmdb_vote_count", sa.Integer()),
|
||||||
|
("budget", sa.BigInteger()),
|
||||||
|
("revenue", sa.BigInteger()),
|
||||||
|
("keywords", postgresql.JSONB()),
|
||||||
|
("crew", postgresql.JSONB()),
|
||||||
|
("studios", postgresql.JSONB()),
|
||||||
|
("production_countries", postgresql.JSONB()),
|
||||||
|
("recommendations", postgresql.JSONB()),
|
||||||
|
("backdrops", postgresql.JSONB()),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
for t in _TABLES:
|
||||||
|
for name, type_ in _COLS:
|
||||||
|
op.add_column(t, sa.Column(name, type_, nullable=True))
|
||||||
|
op.add_column("plex_people", sa.Column("tmdb_person_id", sa.Integer(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("plex_people", "tmdb_person_id")
|
||||||
|
for t in _TABLES:
|
||||||
|
for name, _ in reversed(_COLS):
|
||||||
|
op.drop_column(t, name)
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
"""plex person TMDB bio columns (E5 S6)
|
||||||
|
|
||||||
|
Additive TMDB-sourced person fields on plex_people — biography, birth/death dates, place of birth,
|
||||||
|
known-for department, also-known-as, and a `known_for` credit pool ([{tmdb_id, media_type, title}]
|
||||||
|
resolved to library rows by guid at serve time) — plus a `tmdb_enriched_at` watermark. Filled the
|
||||||
|
first time a person page is opened (from /person/{tmdb_person_id}), then SERVED FROM THE ROW. The
|
||||||
|
`tmdb_person_id` link itself already exists (0071). ADD-only, all nullable → no table rewrite, no
|
||||||
|
ACCESS EXCLUSIVE lock. `tmdb_enriched_at` starts NULL, so every person is enriched once on first view.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0072_plex_person_bio"
|
||||||
|
down_revision: Union[str, None] = "0071_plex_tmdb_fields"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
_COLS: tuple[tuple[str, sa.types.TypeEngine], ...] = (
|
||||||
|
("biography", sa.Text()),
|
||||||
|
("birthday", sa.String(length=10)),
|
||||||
|
("deathday", sa.String(length=10)),
|
||||||
|
("place_of_birth", sa.Text()),
|
||||||
|
("known_for_department", sa.String(length=64)),
|
||||||
|
("also_known_as", postgresql.JSONB()),
|
||||||
|
("known_for", postgresql.JSONB()),
|
||||||
|
("tmdb_enriched_at", sa.DateTime(timezone=True)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
for name, type_ in _COLS:
|
||||||
|
op.add_column("plex_people", sa.Column(name, type_, nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
for name, _ in reversed(_COLS):
|
||||||
|
op.drop_column("plex_people", name)
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""saved_views.source (ask 4: per-module saved views)
|
||||||
|
|
||||||
|
Saved "smart views" were feed-only; the Plex module now has them too. Add a `source` discriminator
|
||||||
|
(default "feed" so every existing row stays a feed view) and widen the per-user unique name constraint
|
||||||
|
to (user_id, source, name) so a feed view and a Plex view can share a name. `source` is NOT NULL with a
|
||||||
|
server_default, so the ADD is a metadata-only rewrite-free change; the constraint swap is quick on a
|
||||||
|
small per-user table.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0073_saved_view_source"
|
||||||
|
down_revision: Union[str, None] = "0072_plex_person_bio"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"saved_views",
|
||||||
|
sa.Column("source", sa.String(length=16), nullable=False, server_default="feed"),
|
||||||
|
)
|
||||||
|
op.drop_constraint("uq_user_saved_view", "saved_views", type_="unique")
|
||||||
|
op.create_unique_constraint(
|
||||||
|
"uq_user_source_saved_view", "saved_views", ["user_id", "source", "name"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# LOSSY / effectively forward-only: upgrade lets a user hold a feed view and a plex view of the same
|
||||||
|
# name, so re-creating the narrow (user_id, name) constraint can raise IntegrityError on such a pair.
|
||||||
|
# De-dup by dropping the non-feed rows first (they can't exist in the pre-0073 schema anyway) so the
|
||||||
|
# narrow constraint re-applies cleanly for the drift test's downgrade→re-upgrade cycle.
|
||||||
|
op.execute("DELETE FROM saved_views WHERE source <> 'feed'")
|
||||||
|
op.drop_constraint("uq_user_source_saved_view", "saved_views", type_="unique")
|
||||||
|
op.create_unique_constraint("uq_user_saved_view", "saved_views", ["user_id", "name"])
|
||||||
|
op.drop_column("saved_views", "source")
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
# Backend conventions
|
||||||
|
|
||||||
|
The rules below are already followed across `backend/app` — they lived only in a few module
|
||||||
|
docstrings (`demo.py`, `userscope.py`, `routes/_common.py`) until this file collected them (E3 S3,
|
||||||
|
B-core-20). When you add code, match these so the next reader doesn't have to guess "which pattern do
|
||||||
|
I copy".
|
||||||
|
|
||||||
|
## Module layering
|
||||||
|
|
||||||
|
Three layers, by *who* runs the code — not by feature:
|
||||||
|
|
||||||
|
1. **`routes/…` — request-only.** A route module holds HTTP concerns: auth dependencies, request/
|
||||||
|
response models, status-code mapping, serialization. It owns logic that runs *only* on a request.
|
||||||
|
Business logic that a request happens to trigger does **not** live here if anything else runs it too.
|
||||||
|
|
||||||
|
2. **`plex/`, `downloads/`, `sync/` — domain logic shared by more than one caller.** Anything the
|
||||||
|
**scheduler or the worker** runs (not just a request) lives in one of these domain packages, so the
|
||||||
|
route, the worker, and the scheduler share one definition. Examples from this codebase:
|
||||||
|
`downloads/service.py` owns enqueue + the job state transitions (routes call it); `plex/sync.py` is
|
||||||
|
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline.
|
||||||
|
|
||||||
|
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||||
|
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, … are single-file primitives many
|
||||||
|
layers import. They depend on nothing above them (a core module must never import a `routes/…`
|
||||||
|
module).
|
||||||
|
|
||||||
|
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||||
|
*second* caller needs it (a worker as well as a route; a scheduler as well as a route), extract it down
|
||||||
|
a layer — into the relevant domain package, or a flat core module if it's cross-cutting. Don't
|
||||||
|
pre-abstract for a single caller; don't leave shared logic duplicated once there are two.
|
||||||
|
|
||||||
|
## Transaction ownership
|
||||||
|
|
||||||
|
This follows CLAUDE.md's canonical rule verbatim — **"the handler/job-wrapper commits; helpers flush
|
||||||
|
(only `quota` owns its session)"** — so read it that way, not as an exception to it. The commit belongs
|
||||||
|
to the **operation owner**; anything called *inside* another transaction only stages its change
|
||||||
|
(`db.add` / `flush`) and lets the owner commit, so the whole operation is one transaction.
|
||||||
|
`audit.record()` is the reference: it adds the audit row and the caller commits it together with the
|
||||||
|
mutation it records, so an audit entry can never exist without its change (or vice-versa).
|
||||||
|
|
||||||
|
- The **operation owner** is usually the route handler. A `service.*` operation the handler delegates a
|
||||||
|
whole transition to (e.g. `service.cancel`) *is* that owner and commits its one transition — it isn't
|
||||||
|
a mid-operation helper. A helper that owns its OWN short-lived session (the worker's `jobdb` writes,
|
||||||
|
the same carve-out CLAUDE.md grants `quota`) commits that session. Everything else flushes.
|
||||||
|
- `SessionLocal` is `expire_on_commit=False`: after a raw `UPDATE` + commit, an ORM object you already
|
||||||
|
loaded is **stale**. Read the fresh value from the row (RETURNING, or a re-`SELECT`), not the ORM
|
||||||
|
attribute — see `service.decr_ref_count`.
|
||||||
|
|
||||||
|
## Shared counters and status: do it atomically at the DB
|
||||||
|
|
||||||
|
- **Never read-modify-write a shared counter in Python** (`x.count = x.count + 1`). Concurrent sessions
|
||||||
|
lose an update. Use a DB-atomic statement — `col = col + 1`, or `greatest(col - 1, 0)` with
|
||||||
|
`RETURNING` so the caller branches on the post-value under the row lock (`service.incr_ref_count` /
|
||||||
|
`decr_ref_count`).
|
||||||
|
- **Download job status is a `JobStatus` StrEnum** (`downloads/status.py`), not bare strings. The
|
||||||
|
`HOLDING` set defines which states hold a `MediaAsset` (so release accounting stays correct);
|
||||||
|
`STOPPED_BY_ROUTE` is the route-initiated stop set. A worker child's terminal write goes through
|
||||||
|
`service.finalize_if_running` (an atomic `UPDATE … WHERE status='running'`) so a finishing child can't
|
||||||
|
resurrect a job a route just stopped.
|
||||||
|
|
||||||
|
## Config: one home per kind
|
||||||
|
|
||||||
|
- **`app.config.settings`** — boot/env config (DB URL, download root, worker concurrency): read once at
|
||||||
|
startup, not user-tunable at runtime.
|
||||||
|
- **`sysconfig.SPECS`** — the runtime admin knobs (DB-override-or-default). A new admin-tunable value
|
||||||
|
goes in `SPECS`; don't invent a fourth store. (`state.py`'s `maintenance_batch` predates this and is
|
||||||
|
slated to fold into `sysconfig` — do not add more like it.)
|
||||||
|
|
||||||
|
## The gate
|
||||||
|
|
||||||
|
Everything below must be green before a change is done (run via `siftlode check`):
|
||||||
|
`ruff` (backend) · `tsc` ×3 + `eslint` + `prettier` + `knip` (frontend) · `vitest` · `pytest`. The
|
||||||
|
pytest lane runs in `backend/docker-compose.test.yml` — an ephemeral Postgres + the backend tree
|
||||||
|
bind-mounted — so the DB-backed integration tests (the `db` fixture) actually run; `SIFTLODE_TEST_
|
||||||
|
REQUIRE_DB=1` there turns a would-be silent skip into a hard failure. See `docs/development.md`.
|
||||||
@@ -9,6 +9,7 @@ runs are always logged. Secrets are never stored (config changes log the key nam
|
|||||||
mutation it describes, so the trail can't drift from reality (and a rolled-back mutation drops
|
mutation it describes, so the trail can't drift from reality (and a rolled-back mutation drops
|
||||||
its audit row too).
|
its audit row too).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
from sqlalchemy import delete
|
from sqlalchemy import delete
|
||||||
|
|||||||
+6
-9
@@ -41,8 +41,7 @@ log = logging.getLogger("siftlode.auth")
|
|||||||
def google_enabled(db: Session) -> bool:
|
def google_enabled(db: Session) -> bool:
|
||||||
"""Whether Sign in with Google is available (both client id + secret resolve to non-empty)."""
|
"""Whether Sign in with Google is available (both client id + secret resolve to non-empty)."""
|
||||||
return bool(
|
return bool(
|
||||||
sysconfig.get_str(db, "google_client_id")
|
sysconfig.get_str(db, "google_client_id") and sysconfig.get_str(db, "google_client_secret")
|
||||||
and sysconfig.get_str(db, "google_client_secret")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -125,7 +124,9 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
|||||||
# ciphertext, so a token row with no refresh token silently failed to revoke). decrypt_optional so a
|
# ciphertext, so a token row with no refresh token silently failed to revoke). decrypt_optional so a
|
||||||
# TOKEN_ENCRYPTION_KEY mismatch can't 500 the purge — a best-effort revoke degrades to "skip". (B-core-2)
|
# TOKEN_ENCRYPTION_KEY mismatch can't 500 the purge — a best-effort revoke degrades to "skip". (B-core-2)
|
||||||
google_token = (
|
google_token = (
|
||||||
(decrypt_optional(tok.refresh_token_enc) or decrypt_optional(tok.access_token)) if tok else None
|
(decrypt_optional(tok.refresh_token_enc) or decrypt_optional(tok.access_token))
|
||||||
|
if tok
|
||||||
|
else None
|
||||||
)
|
)
|
||||||
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
|
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
|
||||||
db.execute(delete(User).where(User.id == user_id))
|
db.execute(delete(User).where(User.id == user_id))
|
||||||
@@ -197,9 +198,7 @@ def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
|||||||
return user
|
return user
|
||||||
|
|
||||||
|
|
||||||
def optional_current_user(
|
def optional_current_user(request: Request, db: Session = Depends(get_db)) -> User | None:
|
||||||
request: Request, db: Session = Depends(get_db)
|
|
||||||
) -> User | None:
|
|
||||||
"""Like `current_user` but returns None instead of raising 401 when there's no valid session.
|
"""Like `current_user` but returns None instead of raising 401 when there's no valid session.
|
||||||
Used by the bootstrap /api/me probe so the logged-out landing doesn't log a failed request to
|
Used by the bootstrap /api/me probe so the logged-out landing doesn't log a failed request to
|
||||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it)."""
|
the browser console (a non-2xx fetch is a console error no matter how the app handles it)."""
|
||||||
@@ -215,9 +214,7 @@ def require_human(user: User = Depends(current_user)) -> User:
|
|||||||
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
|
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
|
||||||
where there's no scope check to lean on."""
|
where there's no scope check to lean on."""
|
||||||
if user.is_demo:
|
if user.is_demo:
|
||||||
raise HTTPException(
|
raise HTTPException(status_code=403, detail="Not available in the demo account.")
|
||||||
status_code=403, detail="Not available in the demo account."
|
|
||||||
)
|
|
||||||
return user
|
return user
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+16
-1
@@ -57,7 +57,7 @@ class Settings(BaseSettings):
|
|||||||
# VPS Caddy front door. A request from any other peer (someone hitting the app port directly)
|
# VPS Caddy front door. A request from any other peer (someone hitting the app port directly)
|
||||||
# is NOT trusted and its raw socket IP is used, so it can't spoof its rate-limit identity via
|
# is NOT trusted and its raw socket IP is used, so it can't spoof its rate-limit identity via
|
||||||
# XFF. Empty = trust no proxy (use the direct peer) — correct for a directly-exposed / local-dev
|
# XFF. Empty = trust no proxy (use the direct peer) — correct for a directly-exposed / local-dev
|
||||||
# deploy. Comma-separated. See auth._client_ip + the deploy docs.
|
# deploy. Comma-separated. See routes.auth._client_ip + the deploy docs.
|
||||||
trusted_proxy_ips: str = ""
|
trusted_proxy_ips: str = ""
|
||||||
|
|
||||||
# --- Outbound email (onboarding: access-request + approval notices) ---
|
# --- Outbound email (onboarding: access-request + approval notices) ---
|
||||||
@@ -169,6 +169,7 @@ class Settings(BaseSettings):
|
|||||||
@property
|
@property
|
||||||
def player_client_fallback_list(self) -> list[str]:
|
def player_client_fallback_list(self) -> list[str]:
|
||||||
return self._clients(self.download_player_clients_fallback)
|
return self._clients(self.download_player_clients_fallback)
|
||||||
|
|
||||||
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
|
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
|
||||||
download_worker_concurrency: int = 2
|
download_worker_concurrency: int = 2
|
||||||
# How often (empty pending queue) the worker polls for a new job, in seconds.
|
# How often (empty pending queue) the worker polls for a new job, in seconds.
|
||||||
@@ -245,6 +246,20 @@ class Settings(BaseSettings):
|
|||||||
# local disk, so the default is fine there.
|
# local disk, so the default is fine there.
|
||||||
plex_hls_dir: str = ""
|
plex_hls_dir: str = ""
|
||||||
|
|
||||||
|
# --- TMDB enrichment (E5 S2; optional — enriches the Plex catalog from TMDB) ---
|
||||||
|
# TMDB API key (v3 api_key, 32 hex chars). Server-side only; stored encrypted when set via the
|
||||||
|
# admin Config page. Empty → the TMDB enrichment job no-ops. Feeds keywords / crew / budget /
|
||||||
|
# revenue / TMDb score / recommendations / backdrops onto the mirrored Plex rows, matched by the
|
||||||
|
# stored canonical guid, and SERVED FROM THE DB — never a live TMDB call at serve time.
|
||||||
|
tmdb_api_key: str = ""
|
||||||
|
tmdb_api_base: str = "https://api.themoviedb.org/3"
|
||||||
|
# Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default.
|
||||||
|
tmdb_language: str = "en-US"
|
||||||
|
# How often the TMDB enrichment job fetches a batch of not-yet-enriched Plex titles, in minutes.
|
||||||
|
tmdb_enrich_interval_min: int = 120
|
||||||
|
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||||
|
tmdb_enrich_batch: int = 100
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def allowed_email_set(self) -> set[str]:
|
def allowed_email_set(self) -> set[str]:
|
||||||
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}
|
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}
|
||||||
|
|||||||
+1
-1
@@ -12,7 +12,7 @@ class Base(DeclarativeBase):
|
|||||||
|
|
||||||
# Pool sized above the sync-endpoint threadpool (~40) so a burst of short DB sessions can't starve on
|
# Pool sized above the sync-endpoint threadpool (~40) so a burst of short DB sessions can't starve on
|
||||||
# checkout. The image proxies deliberately DON'T hold a connection during their slow external fetch
|
# checkout. The image proxies deliberately DON'T hold a connection during their slow external fetch
|
||||||
# (see routes/plex.py), so this headroom is only for the many brief sessions, not long ones.
|
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones.
|
||||||
engine = create_engine(
|
engine = create_engine(
|
||||||
settings.database_url,
|
settings.database_url,
|
||||||
pool_pre_ping=True,
|
pool_pre_ping=True,
|
||||||
|
|||||||
+12
-7
@@ -4,6 +4,7 @@ Shared by the admin's manual reset endpoint (routes/admin.py) and the scheduled
|
|||||||
(scheduler.py). Lives in the service layer, not routes, so the scheduler can call it without an
|
(scheduler.py). Lives in the service layer, not routes, so the scheduler can call it without an
|
||||||
`app.routes -> app.scheduler` import cycle (the reason its import used to be lazy).
|
`app.routes -> app.scheduler` import cycle (the reason its import used to be lazy).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from sqlalchemy import delete, func, select
|
from sqlalchemy import delete, func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
@@ -49,13 +50,17 @@ def _seed_demo_subscriptions(db: Session, demo: User, n: int = 14) -> int:
|
|||||||
Channel manager isn't empty — gives visitors something to explore and matches the landing-page
|
Channel manager isn't empty — gives visitors something to explore and matches the landing-page
|
||||||
screenshot. Idempotent: clears the demo's subscriptions first, then re-adds the top N."""
|
screenshot. Idempotent: clears the demo's subscriptions first, then re-adds the top N."""
|
||||||
db.execute(delete(Subscription).where(Subscription.user_id == demo.id))
|
db.execute(delete(Subscription).where(Subscription.user_id == demo.id))
|
||||||
top = db.execute(
|
top = (
|
||||||
select(Video.channel_id)
|
db.execute(
|
||||||
.where(Video.channel_id.is_not(None))
|
select(Video.channel_id)
|
||||||
.group_by(Video.channel_id)
|
.where(Video.channel_id.is_not(None))
|
||||||
.order_by(func.count().desc())
|
.group_by(Video.channel_id)
|
||||||
.limit(n)
|
.order_by(func.count().desc())
|
||||||
).scalars().all()
|
.limit(n)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for channel_id in top:
|
for channel_id in top:
|
||||||
db.add(Subscription(user_id=demo.id, channel_id=channel_id))
|
db.add(Subscription(user_id=demo.id, channel_id=channel_id))
|
||||||
return len(top)
|
return len(top)
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ Adapters are stateless singletons; get one with `resolve_adapter(source_kind, so
|
|||||||
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
||||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
@@ -140,8 +141,10 @@ def resolve_adapter(
|
|||||||
def host_is(*domains: str) -> bool:
|
def host_is(*domains: str) -> bool:
|
||||||
return any(_host_is(h, *domains) for h in hosts)
|
return any(_host_is(h, *domains) for h in hosts)
|
||||||
|
|
||||||
if source_kind == "youtube" or "youtube" in extractor or host_is(
|
if (
|
||||||
"youtube.com", "youtu.be", "youtube-nocookie.com"
|
source_kind == "youtube"
|
||||||
|
or "youtube" in extractor
|
||||||
|
or host_is("youtube.com", "youtu.be", "youtube-nocookie.com")
|
||||||
):
|
):
|
||||||
return _YOUTUBE
|
return _YOUTUBE
|
||||||
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ URL may have expired — those just get a locally-generated poster instead).
|
|||||||
Run once per environment:
|
Run once per environment:
|
||||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
|
|||||||
+151
-29
@@ -20,6 +20,7 @@ Also builds the editor **filmstrip** — a single tiled sprite of the source vid
|
|||||||
scrub track. Cached on the source asset's `storyboard_path` (reserved in phase 1) and reusable
|
scrub track. Cached on the source asset's `storyboard_path` (reserved in phase 1) and reusable
|
||||||
for a player hover-preview later.
|
for a player hover-preview later.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
@@ -40,6 +41,7 @@ class EditAborted(Exception):
|
|||||||
|
|
||||||
# --- edit spec ------------------------------------------------------------------------------
|
# --- edit spec ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def normalize_edit_spec(spec: dict | None) -> dict:
|
def normalize_edit_spec(spec: dict | None) -> dict:
|
||||||
"""Canonicalize a raw edit spec: clamp/round trim, coerce crop to ints, drop empties.
|
"""Canonicalize a raw edit spec: clamp/round trim, coerce crop to ints, drop empties.
|
||||||
|
|
||||||
@@ -140,6 +142,7 @@ def clip_duration(spec: dict, source_duration: int | None) -> int | None:
|
|||||||
|
|
||||||
# --- ffmpeg: trim / crop --------------------------------------------------------------------
|
# --- ffmpeg: trim / crop --------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
||||||
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
|
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
|
||||||
|
|
||||||
@@ -153,8 +156,15 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
|||||||
reencode = needs_reencode(spec)
|
reencode = needs_reencode(spec)
|
||||||
|
|
||||||
cmd = [
|
cmd = [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-progress", "pipe:1", "-nostats",
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-progress",
|
||||||
|
"pipe:1",
|
||||||
|
"-nostats",
|
||||||
]
|
]
|
||||||
if start:
|
if start:
|
||||||
cmd += ["-ss", f"{start:.3f}"]
|
cmd += ["-ss", f"{start:.3f}"]
|
||||||
@@ -170,8 +180,16 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
|||||||
if crop:
|
if crop:
|
||||||
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
|
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
|
||||||
cmd += [
|
cmd += [
|
||||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
|
"-c:v",
|
||||||
"-c:a", "aac", "-b:a", "192k",
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"veryfast",
|
||||||
|
"-crf",
|
||||||
|
"20",
|
||||||
|
"-c:a",
|
||||||
|
"aac",
|
||||||
|
"-b:a",
|
||||||
|
"192k",
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
||||||
@@ -204,8 +222,15 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
|||||||
crop = spec.get("crop")
|
crop = spec.get("crop")
|
||||||
prep: dict[Path, str] = {}
|
prep: dict[Path, str] = {}
|
||||||
head = [
|
head = [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-progress", "pipe:1", "-nostats",
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-progress",
|
||||||
|
"pipe:1",
|
||||||
|
"-nostats",
|
||||||
]
|
]
|
||||||
|
|
||||||
if needs_reencode(spec):
|
if needs_reencode(spec):
|
||||||
@@ -218,9 +243,24 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
|||||||
labels.append(f"[v{i}][a{i}]")
|
labels.append(f"[v{i}][a{i}]")
|
||||||
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
||||||
cmd = head + [
|
cmd = head + [
|
||||||
"-i", str(src), "-filter_complex", ";".join(parts),
|
"-i",
|
||||||
"-map", "[v]", "-map", "[a]",
|
str(src),
|
||||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-b:a", "192k",
|
"-filter_complex",
|
||||||
|
";".join(parts),
|
||||||
|
"-map",
|
||||||
|
"[v]",
|
||||||
|
"-map",
|
||||||
|
"[a]",
|
||||||
|
"-c:v",
|
||||||
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"veryfast",
|
||||||
|
"-crf",
|
||||||
|
"20",
|
||||||
|
"-c:a",
|
||||||
|
"aac",
|
||||||
|
"-b:a",
|
||||||
|
"192k",
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
listp = staging / "concat.txt"
|
listp = staging / "concat.txt"
|
||||||
@@ -314,7 +354,9 @@ def _stop(proc: subprocess.Popen) -> None:
|
|||||||
escalate_kill(proc, proc.terminate, proc.kill, grace_s=5, label="ffmpeg")
|
escalate_kill(proc, proc.terminate, proc.kill, grace_s=5, label="ffmpeg")
|
||||||
|
|
||||||
|
|
||||||
def run_ffmpeg(cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S) -> None:
|
def run_ffmpeg(
|
||||||
|
cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S
|
||||||
|
) -> None:
|
||||||
"""Run an ffmpeg edit, reporting 0–100 progress and honouring cooperative cancel.
|
"""Run an ffmpeg edit, reporting 0–100 progress and honouring cooperative cancel.
|
||||||
|
|
||||||
Both pipes are drained by threads (see `_drain_tail`) and the main loop polls on a 1s tick, so
|
Both pipes are drained by threads (see `_drain_tail`) and the main loop polls on a 1s tick, so
|
||||||
@@ -411,9 +453,20 @@ def probe_codecs(path: Path) -> tuple[str | None, str | None]:
|
|||||||
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
|
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
|
||||||
try:
|
try:
|
||||||
out = subprocess.run(
|
out = subprocess.run(
|
||||||
["ffprobe", "-v", "error", "-show_entries", "stream=codec_type,codec_name",
|
[
|
||||||
"-of", "json", str(path)],
|
"ffprobe",
|
||||||
capture_output=True, text=True, timeout=30, check=True,
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-show_entries",
|
||||||
|
"stream=codec_type,codec_name",
|
||||||
|
"-of",
|
||||||
|
"json",
|
||||||
|
str(path),
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
check=True,
|
||||||
).stdout
|
).stdout
|
||||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
|
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
|
||||||
return None, None
|
return None, None
|
||||||
@@ -425,16 +478,29 @@ def probe_duration(path: Path) -> int:
|
|||||||
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
|
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
|
||||||
try:
|
try:
|
||||||
out = subprocess.run(
|
out = subprocess.run(
|
||||||
["ffprobe", "-v", "error", "-show_entries", "format=duration",
|
[
|
||||||
"-of", "default=nw=1:nk=1", str(path)],
|
"ffprobe",
|
||||||
capture_output=True, text=True, timeout=30, check=True,
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-show_entries",
|
||||||
|
"format=duration",
|
||||||
|
"-of",
|
||||||
|
"default=nw=1:nk=1",
|
||||||
|
str(path),
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
check=True,
|
||||||
).stdout.strip()
|
).stdout.strip()
|
||||||
return int(float(out)) if out else 0
|
return int(float(out)) if out else 0
|
||||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool) -> tuple[bool, bool, bool]:
|
def browser_transcode_flags(
|
||||||
|
vcodec: str | None, acodec: str | None, is_mp4: bool
|
||||||
|
) -> tuple[bool, bool, bool]:
|
||||||
"""Decide what (if anything) makes a file universally browser-playable. Returns
|
"""Decide what (if anything) makes a file universally browser-playable. Returns
|
||||||
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
|
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
|
||||||
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
|
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
|
||||||
@@ -443,15 +509,30 @@ def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool
|
|||||||
return (rv or ra or not is_mp4), rv, ra
|
return (rv or ra or not is_mp4), rv, ra
|
||||||
|
|
||||||
|
|
||||||
def build_browser_safe_cmd(src: Path, dest: Path, reencode_video: bool, reencode_audio: bool) -> list[str]:
|
def build_browser_safe_cmd(
|
||||||
|
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool
|
||||||
|
) -> list[str]:
|
||||||
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
||||||
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
||||||
pipe:1` streams machine-readable progress for run_ffmpeg."""
|
pipe:1` streams machine-readable progress for run_ffmpeg."""
|
||||||
cmd = [
|
cmd = [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-progress", "pipe:1", "-nostats", "-i", str(src),
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-progress",
|
||||||
|
"pipe:1",
|
||||||
|
"-nostats",
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
]
|
]
|
||||||
cmd += ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"] if reencode_video else ["-c:v", "copy"]
|
cmd += (
|
||||||
|
["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"]
|
||||||
|
if reencode_video
|
||||||
|
else ["-c:v", "copy"]
|
||||||
|
)
|
||||||
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
||||||
cmd += ["-movflags", "+faststart", str(dest)]
|
cmd += ["-movflags", "+faststart", str(dest)]
|
||||||
return cmd
|
return cmd
|
||||||
@@ -486,25 +567,63 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
|
|||||||
fps = geom["count"] / dur
|
fps = geom["count"] / dur
|
||||||
vf = f"fps={fps:.6f},scale={_SB_TILE_W}:-2,tile={geom['cols']}x{geom['rows']}"
|
vf = f"fps={fps:.6f},scale={_SB_TILE_W}:-2,tile={geom['cols']}x{geom['rows']}"
|
||||||
return [
|
return [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-i", str(src), "-frames:v", "1", "-q:v", "5", "-vf", vf, str(dest),
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-q:v",
|
||||||
|
"5",
|
||||||
|
"-vf",
|
||||||
|
vf,
|
||||||
|
str(dest),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
||||||
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
|
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
|
||||||
return [
|
return [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-ss", f"{max(0.0, at_s):.3f}", "-i", str(src),
|
"-y",
|
||||||
"-frames:v", "1", "-q:v", "3", "-vf", "scale=640:-2", str(dest),
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-ss",
|
||||||
|
f"{max(0.0, at_s):.3f}",
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-q:v",
|
||||||
|
"3",
|
||||||
|
"-vf",
|
||||||
|
"scale=640:-2",
|
||||||
|
str(dest),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
|
def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
|
||||||
"""Decode a downloaded thumbnail (webp/png/…) to a single baseline JPEG frame."""
|
"""Decode a downloaded thumbnail (webp/png/…) to a single baseline JPEG frame."""
|
||||||
return [
|
return [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-i", str(src), "-frames:v", "1", "-update", "1", str(dest),
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-update",
|
||||||
|
"1",
|
||||||
|
str(dest),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -550,7 +669,10 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
|||||||
return None
|
return None
|
||||||
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
|
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
|
||||||
if err is not None:
|
if err is not None:
|
||||||
log.info("thumbnail->jpg conversion failed (%s); poster falls back to a cut frame", str(err)[:120])
|
log.info(
|
||||||
|
"thumbnail->jpg conversion failed (%s); poster falls back to a cut frame",
|
||||||
|
str(err)[:120],
|
||||||
|
)
|
||||||
return None
|
return None
|
||||||
return dest if dest.exists() else None
|
return dest if dest.exists() else None
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ spec shape (all keys optional; defaults applied by `normalize`):
|
|||||||
vcodec "h264" | "vp9" | "av1" | null (preference, best-effort)
|
vcodec "h264" | "vp9" | "av1" | null (preference, best-effort)
|
||||||
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
|
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
|
||||||
@@ -215,8 +216,6 @@ def build_ydl_opts(
|
|||||||
# jpg/png thumbnail directly and converts anything else to PNG (not the crashing mjpeg2jpeg-to-jpg
|
# jpg/png thumbnail directly and converts anything else to PNG (not the crashing mjpeg2jpeg-to-jpg
|
||||||
# path), so it never needed the convertor and does not reintroduce the crash.
|
# path), so it never needed the convertor and does not reintroduce the crash.
|
||||||
if s["embed_thumbnail"]:
|
if s["embed_thumbnail"]:
|
||||||
opts["postprocessors"].append(
|
opts["postprocessors"].append({"key": "EmbedThumbnail", "already_have_thumbnail": True})
|
||||||
{"key": "EmbedThumbnail", "already_have_thumbnail": True}
|
|
||||||
)
|
|
||||||
|
|
||||||
return opts
|
return opts
|
||||||
|
|||||||
+59
-35
@@ -15,6 +15,7 @@ Five passes each run:
|
|||||||
|
|
||||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
@@ -40,9 +41,7 @@ _ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight do
|
|||||||
def _holders(db: Session, asset_id: int) -> list[int]:
|
def _holders(db: Session, asset_id: int) -> list[int]:
|
||||||
return list(
|
return list(
|
||||||
db.execute(
|
db.execute(
|
||||||
select(DownloadJob.user_id)
|
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
|
||||||
.where(DownloadJob.asset_id == asset_id)
|
|
||||||
.distinct()
|
|
||||||
).scalars()
|
).scalars()
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -101,7 +100,9 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
|||||||
pass
|
pass
|
||||||
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
|
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
|
||||||
for d in sorted(
|
for d in sorted(
|
||||||
(p for p in entry.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True
|
(p for p in entry.rglob("*") if p.is_dir()),
|
||||||
|
key=lambda p: len(p.parts),
|
||||||
|
reverse=True,
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
d.rmdir()
|
d.rmdir()
|
||||||
@@ -121,15 +122,19 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
|
|
||||||
# 1. Pre-expiry warnings.
|
# 1. Pre-expiry warnings.
|
||||||
warn_before = now + timedelta(days=grace_days)
|
warn_before = now + timedelta(days=grace_days)
|
||||||
soon = db.execute(
|
soon = (
|
||||||
select(MediaAsset).where(
|
db.execute(
|
||||||
MediaAsset.status == "ready",
|
select(MediaAsset).where(
|
||||||
MediaAsset.gc_notified.is_(False),
|
MediaAsset.status == "ready",
|
||||||
MediaAsset.expires_at.is_not(None),
|
MediaAsset.gc_notified.is_(False),
|
||||||
MediaAsset.expires_at > now,
|
MediaAsset.expires_at.is_not(None),
|
||||||
MediaAsset.expires_at <= warn_before,
|
MediaAsset.expires_at > now,
|
||||||
|
MediaAsset.expires_at <= warn_before,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
).scalars().all()
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for asset in soon:
|
for asset in soon:
|
||||||
_notify_holders(db, asset, "download_expiring")
|
_notify_holders(db, asset, "download_expiring")
|
||||||
asset.gc_notified = True
|
asset.gc_notified = True
|
||||||
@@ -138,13 +143,17 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
||||||
dead = db.execute(
|
dead = (
|
||||||
select(MediaAsset).where(
|
db.execute(
|
||||||
MediaAsset.status.in_(("ready", "error")),
|
select(MediaAsset).where(
|
||||||
MediaAsset.expires_at.is_not(None),
|
MediaAsset.status.in_(("ready", "error")),
|
||||||
MediaAsset.expires_at <= now,
|
MediaAsset.expires_at.is_not(None),
|
||||||
|
MediaAsset.expires_at <= now,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
).scalars().all()
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for asset in dead:
|
for asset in dead:
|
||||||
_delete_asset(db, asset, "download_expired")
|
_delete_asset(db, asset, "download_expired")
|
||||||
expired += 1
|
expired += 1
|
||||||
@@ -154,17 +163,24 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
# 3. LRU eviction over the total cache cap.
|
# 3. LRU eviction over the total cache cap.
|
||||||
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
||||||
if cap > 0:
|
if cap > 0:
|
||||||
total = db.execute(
|
total = (
|
||||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
db.execute(
|
||||||
MediaAsset.status == "ready"
|
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||||
)
|
MediaAsset.status == "ready"
|
||||||
).scalar() or 0
|
)
|
||||||
|
).scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
if total > cap:
|
if total > cap:
|
||||||
candidates = db.execute(
|
candidates = (
|
||||||
select(MediaAsset)
|
db.execute(
|
||||||
.where(MediaAsset.status == "ready")
|
select(MediaAsset)
|
||||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
.where(MediaAsset.status == "ready")
|
||||||
).scalars().all()
|
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for asset in candidates:
|
for asset in candidates:
|
||||||
if total <= cap:
|
if total <= cap:
|
||||||
break
|
break
|
||||||
@@ -176,13 +192,17 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
||||||
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
||||||
# holder is gone; they carry no expires_at, so passes 1-3 never touch them.
|
# holder is gone; they carry no expires_at, so passes 1-3 never touch them.
|
||||||
orphans = db.execute(
|
orphans = (
|
||||||
select(MediaAsset).where(
|
db.execute(
|
||||||
MediaAsset.status == "error",
|
select(MediaAsset).where(
|
||||||
MediaAsset.ref_count == 0,
|
MediaAsset.status == "error",
|
||||||
MediaAsset.rel_path.is_(None),
|
MediaAsset.ref_count == 0,
|
||||||
|
MediaAsset.rel_path.is_(None),
|
||||||
|
)
|
||||||
)
|
)
|
||||||
).scalars().all()
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for asset in orphans:
|
for asset in orphans:
|
||||||
db.delete(asset) # no file to unlink; FK SET NULL clears any stray job.asset_id
|
db.delete(asset) # no file to unlink; FK SET NULL clears any stray job.asset_id
|
||||||
reaped += 1
|
reaped += 1
|
||||||
@@ -194,7 +214,11 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
swept = _sweep_orphan_files(db, settings.download_root)
|
swept = _sweep_orphan_files(db, settings.download_root)
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped, "swept": swept
|
"warned": warned,
|
||||||
|
"expired": expired,
|
||||||
|
"evicted": evicted,
|
||||||
|
"reaped": reaped,
|
||||||
|
"swept": swept,
|
||||||
}
|
}
|
||||||
log.info("download_gc %s", result)
|
log.info("download_gc %s", result)
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
"""Short-lived-session DB helpers + staging paths shared by the worker parent and the child pipeline."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.db import SessionLocal
|
||||||
|
from app.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
|
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
|
||||||
|
|
||||||
|
|
||||||
|
def _set_job(job_id: int, **fields) -> None:
|
||||||
|
with SessionLocal() as db:
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
|
if job is None:
|
||||||
|
return
|
||||||
|
for k, v in fields.items():
|
||||||
|
setattr(job, k, v)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _job_status(job_id: int) -> str | None:
|
||||||
|
with SessionLocal() as db:
|
||||||
|
return db.execute(
|
||||||
|
text("SELECT status FROM download_jobs WHERE id = :id"), {"id": job_id}
|
||||||
|
).scalar()
|
||||||
|
|
||||||
|
|
||||||
|
def _touch_job(job_id: int) -> None:
|
||||||
|
"""Bump a running job's `updated_at` without changing any value — a stall-watchdog heartbeat for
|
||||||
|
phases that legitimately write no progress. Extraction writes nothing, and a multi-client-set
|
||||||
|
fallback re-writes identical values (progress 0 / phase 'downloading'), so ORM `onupdate` would
|
||||||
|
NOT fire and two slow extraction attempts could otherwise cross the stall timeout and be killed
|
||||||
|
mid-extraction though they'd have succeeded."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
db.execute(
|
||||||
|
text("UPDATE download_jobs SET updated_at=now() WHERE id=:id AND status='running'"),
|
||||||
|
{"id": job_id},
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _requeue_if_running(job_id: int, phase: str) -> None:
|
||||||
|
"""Put the job back to 'queued' for a later retry, but ONLY while it is still 'running'. The
|
||||||
|
route flips status to paused/cancelled to stop a job; a blind `status='queued'` here would
|
||||||
|
clobber that and silently revive a job the user just cancelled. The conditional WHERE makes the
|
||||||
|
requeue a no-op unless we still own a running job."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
"UPDATE download_jobs SET status='queued', phase=:phase, updated_at=now() "
|
||||||
|
"WHERE id=:id AND status='running'"
|
||||||
|
),
|
||||||
|
{"id": job_id, "phase": phase},
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_asset_pending(asset_id: int) -> None:
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
if asset and asset.status == "downloading":
|
||||||
|
asset.status = "pending"
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _staging_dir(asset_id: int) -> Path:
|
||||||
|
return Path(settings.download_root) / ".staging" / f"asset-{asset_id}"
|
||||||
|
|
||||||
|
|
||||||
|
def _edit_staging_dir(asset_id: int) -> Path:
|
||||||
|
return Path(settings.download_root) / ".staging" / f"edit-{asset_id}"
|
||||||
@@ -6,6 +6,7 @@ require a short-lived signed **grant** on the file request — because a `<video
|
|||||||
header, the password is exchanged once at `/unlock` for an HMAC grant appended to the file URL,
|
header, the password is exchanged once at `/unlock` for an HMAC grant appended to the file URL,
|
||||||
so the raw password never rides in the media URL (or server logs).
|
so the raw password never rides in the media URL (or server logs).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import secrets
|
import secrets
|
||||||
@@ -27,10 +28,9 @@ def is_expired(link: DownloadLink) -> bool:
|
|||||||
|
|
||||||
# --- signed grants (password-protected file access) ----------------------------------------
|
# --- signed grants (password-protected file access) ----------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def _sign(msg: str) -> str:
|
def _sign(msg: str) -> str:
|
||||||
return hmac.new(
|
return hmac.new(settings.secret_key.encode(), msg.encode(), hashlib.sha256).hexdigest()[:32]
|
||||||
settings.secret_key.encode(), msg.encode(), hashlib.sha256
|
|
||||||
).hexdigest()[:32]
|
|
||||||
|
|
||||||
|
|
||||||
def make_grant(token: str, password_version: int, now: float | None = None) -> str:
|
def make_grant(token: str, password_version: int, now: float | None = None) -> str:
|
||||||
@@ -58,6 +58,7 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
|
|||||||
|
|
||||||
# --- serialization -------------------------------------------------------------------------
|
# --- serialization -------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
||||||
"""What the file owner sees when managing their links."""
|
"""What the file owner sees when managing their links."""
|
||||||
return {
|
return {
|
||||||
@@ -72,7 +73,9 @@ def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def public_meta(link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None) -> dict:
|
def public_meta(
|
||||||
|
link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None
|
||||||
|
) -> dict:
|
||||||
"""Metadata for the public watch page (only ever returned once access is authorized).
|
"""Metadata for the public watch page (only ever returned once access is authorized).
|
||||||
|
|
||||||
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
|
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ This injects per-video OG/Twitter tags into the served HTML so a shared link unf
|
|||||||
video's title, channel and thumbnail. Real browsers ignore the extra tags and hydrate the SPA as
|
video's title, channel and thumbnail. Real browsers ignore the extra tags and hydrate the SPA as
|
||||||
usual. Password-protected / expired / invalid links fall back to the generic card (no metadata leak).
|
usual. Password-protected / expired / invalid links fall back to the generic card (no metadata leak).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import html
|
import html
|
||||||
import re
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -71,9 +72,7 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
|
|||||||
|
|
||||||
def _watch_tags(token: str, db: Session) -> str | None:
|
def _watch_tags(token: str, db: Session) -> str | None:
|
||||||
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
|
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
|
||||||
link = db.execute(
|
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
|
||||||
select(DownloadLink).where(DownloadLink.token == token)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if link is None or linksmod.is_expired(link):
|
if link is None or linksmod.is_expired(link):
|
||||||
return None
|
return None
|
||||||
url = f"{settings.app_base}/watch/{token}"
|
url = f"{settings.app_base}/watch/{token}"
|
||||||
|
|||||||
@@ -0,0 +1,634 @@
|
|||||||
|
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import time
|
||||||
|
from datetime import date, datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yt_dlp
|
||||||
|
|
||||||
|
from app import sysconfig
|
||||||
|
from app.config import settings
|
||||||
|
from app.db import SessionLocal
|
||||||
|
from app.downloads import edit as editmod
|
||||||
|
from app.downloads import errors as dlerrors
|
||||||
|
from app.downloads import adapters, formats, service, storage
|
||||||
|
from app.downloads.jobdb import (
|
||||||
|
_edit_staging_dir,
|
||||||
|
_job_status,
|
||||||
|
_reset_asset_pending,
|
||||||
|
_set_job,
|
||||||
|
_staging_dir,
|
||||||
|
_touch_job,
|
||||||
|
)
|
||||||
|
from app.downloads.status import JobStatus
|
||||||
|
from app.models import DownloadJob, MediaAsset
|
||||||
|
from app.notifications import create_notification
|
||||||
|
from app.titles import normalize_title
|
||||||
|
|
||||||
|
log = logging.getLogger("siftlode.worker")
|
||||||
|
|
||||||
|
|
||||||
|
class _Aborted(Exception):
|
||||||
|
"""Raised inside the progress hook when the job was paused/canceled mid-download."""
|
||||||
|
|
||||||
|
|
||||||
|
# Both progress sources call back FAR faster than the database should be touched: yt-dlp fires its
|
||||||
|
# hook per chunk, and ffmpeg's `-progress pipe:1` emits a block of ~12 lines every ~0.5s while
|
||||||
|
# run_ffmpeg calls its callbacks once per LINE. Every callback here is a round-trip (`_job_status`
|
||||||
|
# opens a session and SELECTs; `_set_job` opens one and UPDATE+COMMITs), so unthrottled they cost
|
||||||
|
# ~10^5 queries on a long job. ONE interval, ONE implementation, used by both paths — three
|
||||||
|
# hand-rolled copies is how the ffmpeg side ended up unthrottled while the download side was fine.
|
||||||
|
_DB_WRITE_INTERVAL_S = 1.0
|
||||||
|
_CANCEL_POLL_INTERVAL_S = 1.0
|
||||||
|
|
||||||
|
_THUMB_SUFFIXES = (".jpg", ".jpeg", ".png", ".webp")
|
||||||
|
# yt-dlp's working files in the staging dir. None of them is ever the finished media, but several
|
||||||
|
# CAN be the biggest file there (a video-only per-format stream, an abandoned fragment run), so the
|
||||||
|
# staging fallback must exclude them by name — picking one would store e.g. a silent video-only
|
||||||
|
# stream as a ready asset.
|
||||||
|
_YTDLP_TEMP_RE = re.compile(
|
||||||
|
r"\.part(-Frag\d+)?$" # in-flight download / fragment
|
||||||
|
r"|\.ytdl$" # resume state
|
||||||
|
r"|\.temp\.[^.]+$" # merge/postprocess temp
|
||||||
|
r"|\.f\d+\.[^.]+$", # per-format stream kept before the merge
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *, kind: str) -> None:
|
||||||
|
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
||||||
|
silently while scheduler jobs and playlist-push already notified (R11 feedback symmetry). A
|
||||||
|
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
||||||
|
scheduler._notify_done). `kind` ("download"/"edit") lets the inbox pick accurate copy; the
|
||||||
|
English title here is only a fallback for an unknown type. `user_id`/`title` are captured by the
|
||||||
|
caller BEFORE its commit, so reading them here needs no post-commit refresh of an expired row."""
|
||||||
|
if user_id is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
create_notification(
|
||||||
|
db,
|
||||||
|
user_id=user_id,
|
||||||
|
type="download_done",
|
||||||
|
title=f"Ready to watch: {title}" if title else "Ready to watch",
|
||||||
|
data={"title": title, "kind": kind},
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
db.rollback()
|
||||||
|
log.exception("failed to write download-done notification for job %s", job_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_upload_date(raw) -> date | None:
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.strptime(str(raw), "%Y%m%d").date()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||||
|
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
|
||||||
|
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
|
||||||
|
real title + thumbnail while it downloads, not just the URL. Zero extra network cost."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
if asset is None or asset.title:
|
||||||
|
return
|
||||||
|
# `or info.get("id")` avoids a blank card while downloading when the cleaned title is empty
|
||||||
|
# (e.g. a title that was only a social engagement prefix); the completion path later sets
|
||||||
|
# the final title with its own source_ref fallback.
|
||||||
|
asset.title = normalize_title(info.get("title")) or info.get("id")
|
||||||
|
asset.uploader = info.get("uploader") or info.get("channel")
|
||||||
|
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
|
||||||
|
asset.thumbnail_url = info.get("thumbnail")
|
||||||
|
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
||||||
|
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
||||||
|
asset.source_webpage_url = info.get("webpage_url")
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _make_progress_hook(job_id: int, asset_id: int):
|
||||||
|
state = {"meta_done": False}
|
||||||
|
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
|
||||||
|
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S)
|
||||||
|
|
||||||
|
def hook(d: dict) -> None:
|
||||||
|
# Cooperative pause/cancel: if the job is no longer 'running', abort the download.
|
||||||
|
if cancelled():
|
||||||
|
raise _Aborted
|
||||||
|
if not state["meta_done"] and d.get("info_dict"):
|
||||||
|
state["meta_done"] = True
|
||||||
|
try:
|
||||||
|
_fill_asset_meta_early(asset_id, d["info_dict"])
|
||||||
|
except Exception: # noqa: BLE001 — metadata is best-effort, never fail a download
|
||||||
|
pass
|
||||||
|
if d.get("status") != "downloading":
|
||||||
|
return
|
||||||
|
if not allow_write():
|
||||||
|
return
|
||||||
|
# yt-dlp downloads the video and audio streams SEPARATELY (each 0→100%), then merges.
|
||||||
|
# Label the phase so the user understands the bar resetting between streams instead of
|
||||||
|
# seeing a mysterious 95%→5% jump.
|
||||||
|
info = d.get("info_dict") or {}
|
||||||
|
vcodec = info.get("vcodec") or "none"
|
||||||
|
acodec = info.get("acodec") or "none"
|
||||||
|
if vcodec != "none" and acodec == "none":
|
||||||
|
phase = "video"
|
||||||
|
elif acodec != "none" and vcodec == "none":
|
||||||
|
phase = "audio"
|
||||||
|
else:
|
||||||
|
phase = "media"
|
||||||
|
total = d.get("total_bytes") or d.get("total_bytes_estimate") or 0
|
||||||
|
done = d.get("downloaded_bytes") or 0
|
||||||
|
pct = int(done * 100 / total) if total else 0
|
||||||
|
_set_job(
|
||||||
|
job_id,
|
||||||
|
progress=min(pct, 99),
|
||||||
|
speed_bps=int(d["speed"]) if d.get("speed") else None,
|
||||||
|
eta_s=int(d["eta"]) if d.get("eta") else None,
|
||||||
|
phase=phase,
|
||||||
|
)
|
||||||
|
|
||||||
|
return hook
|
||||||
|
|
||||||
|
|
||||||
|
def _rate_limiter(interval_s: float, *, now=time.monotonic):
|
||||||
|
"""`should_run()` → True at most once per `interval_s` (the first call always passes).
|
||||||
|
`now` is injectable so tests drive a clock instead of patching the global one."""
|
||||||
|
# `None`, not 0.0: "never ran" has to be a VALUE, not a falsy timestamp. A truthiness test here
|
||||||
|
# reads a legitimate `now() == 0.0` as "never ran" and lets a second call straight through —
|
||||||
|
# unreachable with real `time.monotonic()`, but `now` exists to be injected, and a test clock
|
||||||
|
# starting at 0 is the obvious choice. That would let a throttle test encode the wrong
|
||||||
|
# behaviour and still pass.
|
||||||
|
state: dict[str, float | None] = {"at": None}
|
||||||
|
|
||||||
|
def should_run() -> bool:
|
||||||
|
t = now()
|
||||||
|
last = state["at"]
|
||||||
|
if last is not None and t - last < interval_s:
|
||||||
|
return False
|
||||||
|
state["at"] = t
|
||||||
|
return True
|
||||||
|
|
||||||
|
return should_run
|
||||||
|
|
||||||
|
|
||||||
|
def _cancel_poll(job_id: int, interval_s: float = _CANCEL_POLL_INTERVAL_S, *, now=time.monotonic):
|
||||||
|
"""Cooperative "has this job been paused/cancelled?" poll, rate-limited and sticky: once it
|
||||||
|
says yes it never queries again. Shared by the yt-dlp hook and run_ffmpeg so cancel latency is
|
||||||
|
one policy, not one per call site."""
|
||||||
|
allow = _rate_limiter(interval_s, now=now)
|
||||||
|
state = {"cancelled": False}
|
||||||
|
|
||||||
|
def cancelled() -> bool:
|
||||||
|
if not state["cancelled"] and allow():
|
||||||
|
state["cancelled"] = _job_status(job_id) not in ("running", None)
|
||||||
|
return state["cancelled"]
|
||||||
|
|
||||||
|
return cancelled
|
||||||
|
|
||||||
|
|
||||||
|
def _ffmpeg_progress_writer(job_id: int, phase: str, *, now=time.monotonic):
|
||||||
|
"""`on_progress` for run_ffmpeg: persist a progress row write at most once per interval and only
|
||||||
|
when the whole-percent value changed (the bar shows integers), plus a separate once-per-interval
|
||||||
|
heartbeat while the percent is stuck (so a slow re-encode isn't mistaken for a stall)."""
|
||||||
|
# Progress writes have their OWN 1/interval budget so a heartbeat can never delay a real write (the
|
||||||
|
# round-1 concern). The heartbeat, in turn, is gated on `last_touch` — the last DB write OR beat —
|
||||||
|
# so a progress write (which already refreshes updated_at) suppresses a redundant heartbeat that
|
||||||
|
# same interval, keeping active downloads at ~1 write/interval, not two.
|
||||||
|
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S, now=now)
|
||||||
|
state = {"pct": -1}
|
||||||
|
last_touch: dict = {"t": None} # monotonic time of the last DB write/heartbeat
|
||||||
|
|
||||||
|
def write(pct: float) -> None:
|
||||||
|
whole = int(pct)
|
||||||
|
if whole != state["pct"]:
|
||||||
|
if allow_write():
|
||||||
|
state["pct"] = whole
|
||||||
|
_set_job(job_id, progress=whole, phase=phase)
|
||||||
|
last_touch["t"] = now()
|
||||||
|
return
|
||||||
|
# Same integer percent, but ffmpeg IS still working — a slow re-encode (AV1/VP9 -> H.264 or a
|
||||||
|
# crop on the CPU-only host) can sit on one percent for minutes. Heartbeat updated_at so the
|
||||||
|
# stall watchdog doesn't kill a live job (B-sub-4) — but only if no write/beat already touched
|
||||||
|
# the row this interval, so a progressing download doesn't pay a redundant write.
|
||||||
|
t = now()
|
||||||
|
if last_touch["t"] is None or t - last_touch["t"] >= _DB_WRITE_INTERVAL_S:
|
||||||
|
_touch_job(job_id)
|
||||||
|
last_touch["t"] = t
|
||||||
|
|
||||||
|
return write
|
||||||
|
|
||||||
|
|
||||||
|
def _pp_phase(pp: str) -> str:
|
||||||
|
"""Map a yt-dlp postprocessor class name to a user-facing phase label. ffmpeg post-steps
|
||||||
|
aren't byte-progress operations (no %), so the UI shows the step name + an indeterminate
|
||||||
|
pulse — this makes the long merge/finalize legible instead of a silent 'Processing'."""
|
||||||
|
if "Merg" in pp:
|
||||||
|
return "merging"
|
||||||
|
if "ExtractAudio" in pp:
|
||||||
|
return "audio_extract"
|
||||||
|
if "Thumbnail" in pp:
|
||||||
|
return "thumbnail"
|
||||||
|
if "SponsorBlock" in pp or "ModifyChapters" in pp:
|
||||||
|
return "sponsorblock"
|
||||||
|
if "Metadata" in pp:
|
||||||
|
return "metadata"
|
||||||
|
return "processing"
|
||||||
|
|
||||||
|
|
||||||
|
def _make_pp_hook(job_id: int):
|
||||||
|
"""Postprocessor hook: surface the current ffmpeg step so the bar shows a concrete label
|
||||||
|
(Merging / Embedding thumbnail / …) instead of freezing at 100% after the streams finish."""
|
||||||
|
|
||||||
|
def hook(d: dict) -> None:
|
||||||
|
if d.get("status") in ("started", "processing"):
|
||||||
|
_set_job(
|
||||||
|
job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None
|
||||||
|
)
|
||||||
|
|
||||||
|
return hook
|
||||||
|
|
||||||
|
|
||||||
|
def _find_thumbnail(staging: Path, media: Path) -> Path | None:
|
||||||
|
for p in staging.iterdir():
|
||||||
|
if p != media and p.suffix.lower() in _THUMB_SUFFIXES:
|
||||||
|
return p
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _produced_file(info: dict, staging: Path) -> Path:
|
||||||
|
"""The media file yt-dlp actually wrote.
|
||||||
|
|
||||||
|
`requested_downloads[0].filepath` is the authoritative answer on YouTube, but an arbitrary-URL
|
||||||
|
extractor may not fill it in at all — and `Path("")` is `Path(".")`, which *passes* `.exists()`,
|
||||||
|
so a missing value used to sail past the fallback and blow up two steps later (`with_name` on an
|
||||||
|
empty name: the BBC-news failure). Anything that isn't a real file counts as absent; then fall
|
||||||
|
back to the largest file left in staging that is neither a thumbnail nor one of yt-dlp's own
|
||||||
|
working files (see `_YTDLP_TEMP_RE`)."""
|
||||||
|
req = (info.get("requested_downloads") or [{}])[0]
|
||||||
|
fp = (
|
||||||
|
req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
|
||||||
|
)
|
||||||
|
if fp:
|
||||||
|
produced = Path(str(fp))
|
||||||
|
if produced.is_file():
|
||||||
|
return produced
|
||||||
|
cands = [
|
||||||
|
p
|
||||||
|
for p in staging.iterdir()
|
||||||
|
if p.is_file()
|
||||||
|
and p.suffix.lower() not in _THUMB_SUFFIXES
|
||||||
|
and not _YTDLP_TEMP_RE.search(p.name)
|
||||||
|
]
|
||||||
|
if not cands:
|
||||||
|
raise RuntimeError("yt-dlp produced no output file")
|
||||||
|
return max(cands, key=lambda p: p.stat().st_size)
|
||||||
|
|
||||||
|
|
||||||
|
def _media_info(info: dict) -> dict:
|
||||||
|
"""Unwrap a playlist/multi_video extraction to the entry that carries the download.
|
||||||
|
|
||||||
|
A single non-YouTube page (a news article's video, a multi-rendition embed) can extract as a
|
||||||
|
wrapper whose own dict has no `requested_downloads` — the media lives on an entry. Pick the
|
||||||
|
FIRST entry that actually downloaded something: on a page with a skipped teaser in front of the
|
||||||
|
real video, entries[0] carries no file, and taking it blindly would label the produced file
|
||||||
|
with the wrong id/title/duration."""
|
||||||
|
entries = [e for e in (info.get("entries") or []) if e]
|
||||||
|
if not entries or info.get("requested_downloads"):
|
||||||
|
return info
|
||||||
|
chosen = next((e for e in entries if e.get("requested_downloads")), entries[0])
|
||||||
|
return {**info, **chosen}
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||||
|
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||||
|
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||||
|
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
|
||||||
|
client_sets = adapter.client_sets()
|
||||||
|
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||||
|
last_exc: Exception | None = None
|
||||||
|
for i, clients in enumerate(client_sets):
|
||||||
|
opts = formats.build_ydl_opts(
|
||||||
|
spec,
|
||||||
|
outtmpl,
|
||||||
|
_make_progress_hook(job_id, asset_id),
|
||||||
|
extra=adapter.ydl_opts_extra(clients),
|
||||||
|
)
|
||||||
|
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||||
|
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||||
|
# multi-client-set fallback would otherwise cross the stall timeout with a stale updated_at.
|
||||||
|
# (Per-ATTEMPT only; a single extract_info that itself runs past the stall timeout is still
|
||||||
|
# killed — acceptable, as socket_timeout + extractor_retries bound one attempt well under it.)
|
||||||
|
try:
|
||||||
|
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||||
|
return ydl.extract_info(url, download=True)
|
||||||
|
except yt_dlp.utils.DownloadError as exc:
|
||||||
|
last_exc = exc
|
||||||
|
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||||
|
log.info(
|
||||||
|
"job %s: clients %s failed (%s); retrying with fallback",
|
||||||
|
job_id,
|
||||||
|
clients,
|
||||||
|
str(exc)[:70],
|
||||||
|
)
|
||||||
|
for p in list(staging.iterdir()): # drop partials before the retry
|
||||||
|
try:
|
||||||
|
p.unlink()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
_set_job(job_id, progress=0, phase="downloading")
|
||||||
|
continue
|
||||||
|
raise
|
||||||
|
raise last_exc # unreachable (loop either returns or raises)
|
||||||
|
|
||||||
|
|
||||||
|
def _download_settings() -> tuple[str, int]:
|
||||||
|
"""Layout + retention days from the DB config (admin-overridable on the Configuration page),
|
||||||
|
falling back to the env defaults. Read here, not from `settings`, so an admin edit takes effect."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
return (
|
||||||
|
sysconfig.get_str(db, "download_layout"),
|
||||||
|
sysconfig.get_int(db, "download_retention_days"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict) -> Path:
|
||||||
|
"""Make an mp4-compat-profile download play in every browser: H.264 + AAC/MP3 in mp4. If the
|
||||||
|
download (whatever the source shipped) isn't already that, re-encode the offending stream(s) —
|
||||||
|
or just remux when the codecs are fine but the container isn't. **Scope:** only the mp4 compat
|
||||||
|
profile is normalized; an explicit `vcodec` or a non-mp4 custom profile is the user opting out
|
||||||
|
of the guarantee and is left as-is (so a `webm` profile can still ship VP9/Opus). Returns the
|
||||||
|
file to store (converted, or the original) and corrects `info`'s codec fields to match disk."""
|
||||||
|
if spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||||
|
return produced
|
||||||
|
vcodec, acodec = editmod.probe_codecs(produced)
|
||||||
|
is_mp4 = produced.suffix.lower() == ".mp4"
|
||||||
|
needs, reenc_v, reenc_a = editmod.browser_transcode_flags(vcodec, acodec, is_mp4)
|
||||||
|
if not needs:
|
||||||
|
return produced
|
||||||
|
|
||||||
|
dest = produced.with_name(produced.stem + ".browser.mp4")
|
||||||
|
action = "re-encode" if (reenc_v or reenc_a) else "remux"
|
||||||
|
log.info(
|
||||||
|
"job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4
|
||||||
|
)
|
||||||
|
_set_job(job_id, phase="processing", progress=0, speed_bps=None, eta_s=None)
|
||||||
|
# Prefer the info duration; probe the file when the extractor didn't provide one, so a long
|
||||||
|
# re-encode still shows real progress instead of a frozen 0%.
|
||||||
|
total_s = int(info["duration"]) if info.get("duration") else editmod.probe_duration(produced)
|
||||||
|
editmod.run_ffmpeg(
|
||||||
|
editmod.build_browser_safe_cmd(produced, dest, reenc_v, reenc_a),
|
||||||
|
total_s=total_s,
|
||||||
|
on_progress=_ffmpeg_progress_writer(job_id, "processing"),
|
||||||
|
should_cancel=_cancel_poll(job_id),
|
||||||
|
)
|
||||||
|
produced.unlink(missing_ok=True)
|
||||||
|
# Keep the stored asset metadata truthful about what's now on disk.
|
||||||
|
if reenc_v:
|
||||||
|
info["vcodec"] = "h264"
|
||||||
|
if reenc_a:
|
||||||
|
info["acodec"] = "aac"
|
||||||
|
return dest
|
||||||
|
|
||||||
|
|
||||||
|
def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict) -> None:
|
||||||
|
staging = _staging_dir(asset_id)
|
||||||
|
try:
|
||||||
|
if staging.exists():
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
staging.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
_set_job(job_id, phase="downloading", progress=0)
|
||||||
|
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
||||||
|
url = service.source_url(source_kind, source_ref)
|
||||||
|
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
|
||||||
|
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
||||||
|
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
||||||
|
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||||
|
|
||||||
|
produced = _produced_file(info, staging)
|
||||||
|
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||||
|
ext = produced.suffix.lstrip(".").lower()
|
||||||
|
meta = storage.MediaMeta(
|
||||||
|
video_id=info.get("id") or source_ref,
|
||||||
|
title=normalize_title(info.get("title")) or source_ref,
|
||||||
|
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
||||||
|
upload_date=_parse_upload_date(info.get("upload_date")),
|
||||||
|
uniqueid_type=adapter.nfo_uniqueid_type(),
|
||||||
|
duration_s=int(info["duration"]) if info.get("duration") else None,
|
||||||
|
description=info.get("description"),
|
||||||
|
thumbnail_url=info.get("thumbnail"),
|
||||||
|
)
|
||||||
|
|
||||||
|
layout, retention_days = _download_settings()
|
||||||
|
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
|
||||||
|
thumb = _find_thumbnail(staging, produced)
|
||||||
|
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
|
||||||
|
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
|
||||||
|
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
|
||||||
|
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
|
||||||
|
storage.place_file(produced, settings.download_root, rel)
|
||||||
|
nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, poster_jpg)
|
||||||
|
|
||||||
|
final = storage.abs_path(settings.download_root, rel)
|
||||||
|
size = final.stat().st_size if final.exists() else None
|
||||||
|
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||||
|
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
if asset:
|
||||||
|
asset.status = "ready"
|
||||||
|
asset.rel_path = rel
|
||||||
|
asset.size_bytes = size
|
||||||
|
asset.duration_s = meta.duration_s
|
||||||
|
asset.width = int(info["width"]) if info.get("width") else None
|
||||||
|
asset.height = int(info["height"]) if info.get("height") else None
|
||||||
|
asset.container = ext
|
||||||
|
asset.vcodec = (
|
||||||
|
info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||||
|
)
|
||||||
|
asset.acodec = (
|
||||||
|
info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||||
|
)
|
||||||
|
asset.title = meta.title
|
||||||
|
asset.uploader = meta.uploader
|
||||||
|
asset.uploader_url = adapter.uploader_url(info)
|
||||||
|
asset.upload_date = meta.upload_date
|
||||||
|
asset.thumbnail_url = meta.thumbnail_url
|
||||||
|
# Record a self-hosted poster for every download: it's the reliable link-preview
|
||||||
|
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
|
||||||
|
# and can't be relied on cross-origin. write_sidecars already wrote <base>.jpg from
|
||||||
|
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
|
||||||
|
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||||
|
asset.source_webpage_url = info.get("webpage_url")
|
||||||
|
asset.nfo_written = nfo_ok
|
||||||
|
asset.error = None
|
||||||
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
|
asset.expires_at = expires
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
|
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
||||||
|
owner_id = job.user_id if job else None
|
||||||
|
fields = {
|
||||||
|
"status": JobStatus.DONE,
|
||||||
|
"progress": 100,
|
||||||
|
"phase": None,
|
||||||
|
"speed_bps": None,
|
||||||
|
"eta_s": None,
|
||||||
|
}
|
||||||
|
if job and not job.display_name and meta.title:
|
||||||
|
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
||||||
|
fields["display_name"] = meta.title[:255]
|
||||||
|
# Atomic terminal write: finalizes ONLY while the row is still running, so a child that
|
||||||
|
# finishes just after a route cancel/pause can't resurrect it (see finalize_if_running).
|
||||||
|
finalized = service.finalize_if_running(db, job_id, **fields)
|
||||||
|
db.commit()
|
||||||
|
if finalized:
|
||||||
|
_notify_download_done(db, owner_id, meta.title, job_id, kind="download")
|
||||||
|
log.info("job %s done: %s", job_id, rel)
|
||||||
|
|
||||||
|
except (_Aborted, editmod.EditAborted):
|
||||||
|
# Paused/canceled mid-download or mid-transcode: free the asset for a later retry; keep the
|
||||||
|
# job's status (the route set it to paused/canceled). ref_count is adjusted by the route.
|
||||||
|
_reset_asset_pending(asset_id)
|
||||||
|
log.info("job %s aborted (pause/cancel)", job_id)
|
||||||
|
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
|
||||||
|
msg = str(exc)[:500]
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
if asset:
|
||||||
|
asset.status = "error"
|
||||||
|
asset.error = msg
|
||||||
|
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
|
||||||
|
# with `error` (error also holds the asset → would re-open the double-release class).
|
||||||
|
service.finalize_if_running(
|
||||||
|
db,
|
||||||
|
job_id,
|
||||||
|
status=JobStatus.ERROR,
|
||||||
|
error=msg,
|
||||||
|
error_code=dlerrors.classify(msg),
|
||||||
|
phase=None,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
log.warning("job %s failed: %s", job_id, msg)
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
if asset:
|
||||||
|
asset.status = "error"
|
||||||
|
asset.error = msg
|
||||||
|
# Atomic fail: only while still running (see the download error path).
|
||||||
|
service.finalize_if_running(
|
||||||
|
db,
|
||||||
|
job_id,
|
||||||
|
status=JobStatus.ERROR,
|
||||||
|
error=msg,
|
||||||
|
error_code=dlerrors.classify(msg),
|
||||||
|
phase=None,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
log.warning("edit job %s failed: %s", job_id, msg)
|
||||||
|
|
||||||
|
|
||||||
|
def _process_edit(job_id: int, asset_id: int) -> None:
|
||||||
|
"""Produce a per-user trim/crop clip from the source asset's downloaded file via ffmpeg."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
|
src = db.get(MediaAsset, job.source_asset_id) if job and job.source_asset_id else None
|
||||||
|
edit_spec = dict(job.edit_spec or {}) if job else {}
|
||||||
|
user_id = job.user_id if job else None
|
||||||
|
src_ready = src is not None and src.status == "ready" and bool(src.rel_path)
|
||||||
|
src_rel = src.rel_path if src else None
|
||||||
|
src_w, src_h, src_dur = (
|
||||||
|
(src.width, src.height, src.duration_s) if src else (None, None, None)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not src_ready or not src_rel:
|
||||||
|
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
||||||
|
return
|
||||||
|
src_path = storage.abs_path(settings.download_root, src_rel)
|
||||||
|
if not src_path.exists():
|
||||||
|
_fail_edit(job_id, asset_id, "The source file is missing.")
|
||||||
|
return
|
||||||
|
|
||||||
|
src_ext = src_path.suffix.lstrip(".").lower() or "mp4"
|
||||||
|
crop = editmod.has_crop(edit_spec)
|
||||||
|
out_ext = "mp4" if editmod.needs_reencode(edit_spec) else src_ext
|
||||||
|
total_s = editmod.clip_duration(edit_spec, src_dur) or src_dur or 0
|
||||||
|
staging = _edit_staging_dir(asset_id)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if staging.exists():
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
staging.mkdir(parents=True, exist_ok=True)
|
||||||
|
dest_staging = staging / f"out.{out_ext}"
|
||||||
|
|
||||||
|
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
||||||
|
if edit_spec.get("segments"):
|
||||||
|
cmd, prep = editmod.build_concat_plan(
|
||||||
|
src_path, dest_staging, edit_spec, out_ext, staging
|
||||||
|
)
|
||||||
|
for p, content in prep.items():
|
||||||
|
p.write_text(content, encoding="utf-8")
|
||||||
|
else:
|
||||||
|
cmd = editmod.build_edit_cmd(src_path, dest_staging, edit_spec, out_ext)
|
||||||
|
editmod.run_ffmpeg(
|
||||||
|
cmd,
|
||||||
|
total_s,
|
||||||
|
on_progress=_ffmpeg_progress_writer(job_id, "editing"),
|
||||||
|
should_cancel=_cancel_poll(job_id),
|
||||||
|
)
|
||||||
|
if not dest_staging.exists():
|
||||||
|
raise RuntimeError("ffmpeg produced no output file")
|
||||||
|
|
||||||
|
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||||
|
storage.place_file(dest_staging, settings.download_root, rel)
|
||||||
|
final = storage.abs_path(settings.download_root, rel)
|
||||||
|
size = final.stat().st_size if final.exists() else None
|
||||||
|
_, retention_days = _download_settings()
|
||||||
|
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||||
|
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
if asset:
|
||||||
|
asset.status = "ready"
|
||||||
|
asset.rel_path = rel
|
||||||
|
asset.size_bytes = size
|
||||||
|
asset.container = out_ext
|
||||||
|
asset.duration_s = total_s or asset.duration_s
|
||||||
|
if crop:
|
||||||
|
asset.width = edit_spec["crop"]["w"]
|
||||||
|
asset.height = edit_spec["crop"]["h"]
|
||||||
|
else:
|
||||||
|
asset.width, asset.height = src_w, src_h
|
||||||
|
asset.error = None
|
||||||
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
|
asset.expires_at = expires
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
|
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
|
||||||
|
owner_id = job.user_id if job else None
|
||||||
|
dl_title = asset.title if asset else None
|
||||||
|
# Atomic terminal write (see the download path): don't resurrect a route-stopped job.
|
||||||
|
finalized = service.finalize_if_running(
|
||||||
|
db, job_id, status=JobStatus.DONE, progress=100, phase=None
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
if finalized:
|
||||||
|
_notify_download_done(db, owner_id, dl_title, job_id, kind="edit")
|
||||||
|
log.info("edit job %s done: %s", job_id, rel)
|
||||||
|
|
||||||
|
except editmod.EditAborted:
|
||||||
|
# Paused/canceled mid-encode: free the asset for a later retry (ref_count is the route's job).
|
||||||
|
_reset_asset_pending(asset_id)
|
||||||
|
log.info("edit job %s aborted (pause/cancel)", job_id)
|
||||||
|
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||||
|
_fail_edit(job_id, asset_id, str(exc)[:500])
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
@@ -8,6 +8,7 @@ Limits come from a per-user DownloadQuota row if the admin set one, else the sys
|
|||||||
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
|
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
|
||||||
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
|
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ Contract: the asset is already claimed ('downloading') by the parent. This proce
|
|||||||
writes the terminal job/asset state itself (done / error, or a cooperative pause via the in-flight
|
writes the terminal job/asset state itself (done / error, or a cooperative pause via the in-flight
|
||||||
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
@@ -38,8 +39,8 @@ def main() -> None:
|
|||||||
return
|
return
|
||||||
|
|
||||||
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
||||||
# real work, and to keep the import graph obvious: runner is a thin entry over worker's helpers.
|
# real work, and to keep the import graph obvious: runner is a thin entry over the pipeline.
|
||||||
from app.worker import _download, _process_edit
|
from app.downloads.pipeline import _download, _process_edit
|
||||||
|
|
||||||
if job_kind == "edit":
|
if job_kind == "edit":
|
||||||
_process_edit(job_id, asset_id)
|
_process_edit(job_id, asset_id)
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ cache hit (asset already ready) — marks the job done instantly with no re-down
|
|||||||
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
|
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
|
||||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import func, select, update
|
||||||
@@ -112,6 +113,34 @@ def next_queue_pos(db: Session) -> int:
|
|||||||
return (db.execute(select(func.coalesce(func.max(DownloadJob.queue_pos), 0))).scalar() or 0) + 1
|
return (db.execute(select(func.coalesce(func.max(DownloadJob.queue_pos), 0))).scalar() or 0) + 1
|
||||||
|
|
||||||
|
|
||||||
|
def incr_ref_count(db: Session, asset_id: int) -> None:
|
||||||
|
"""Atomically +1 an asset's hold count AT THE DB. A Python read-modify-write
|
||||||
|
(`asset.ref_count = asset.ref_count + 1`) loses an update when two enqueues of the same
|
||||||
|
(source,format) run in parallel — both read the same value and write the same +1 — leaving the
|
||||||
|
count one short, so the first release can free a file another job still holds (B-sub-5)."""
|
||||||
|
db.execute(
|
||||||
|
update(MediaAsset)
|
||||||
|
.where(MediaAsset.id == asset_id)
|
||||||
|
.values(ref_count=MediaAsset.ref_count + 1)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def decr_ref_count(db: Session, asset_id: int) -> int:
|
||||||
|
"""Atomically -1 an asset's hold count (clamped at 0) and RETURN the new value. The caller decides
|
||||||
|
whether to free the asset from the returned count while still holding the row's write lock, so two
|
||||||
|
concurrent releases can't both observe the same pre-decrement value and double-free — Postgres
|
||||||
|
serialises the two UPDATEs on the row lock (B-sub-5). synchronize_session=False: the ORM copy is
|
||||||
|
intentionally left stale; callers read the returned value, not `asset.ref_count`."""
|
||||||
|
return db.execute(
|
||||||
|
update(MediaAsset)
|
||||||
|
.where(MediaAsset.id == asset_id)
|
||||||
|
.values(ref_count=func.greatest(MediaAsset.ref_count - 1, 0))
|
||||||
|
.returning(MediaAsset.ref_count)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
|
||||||
def enqueue(
|
def enqueue(
|
||||||
db: Session,
|
db: Session,
|
||||||
user_id: int,
|
user_id: int,
|
||||||
@@ -138,12 +167,13 @@ def enqueue(
|
|||||||
source_ref=source_ref,
|
source_ref=source_ref,
|
||||||
profile_id=profile_id,
|
profile_id=profile_id,
|
||||||
profile_snapshot=spec,
|
profile_snapshot=spec,
|
||||||
display_name=display_name or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
display_name=display_name
|
||||||
|
or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
||||||
status="queued",
|
status="queued",
|
||||||
queue_pos=next_queue_pos(db),
|
queue_pos=next_queue_pos(db),
|
||||||
)
|
)
|
||||||
db.add(job)
|
db.add(job)
|
||||||
asset.ref_count = (asset.ref_count or 0) + 1
|
incr_ref_count(db, asset.id)
|
||||||
|
|
||||||
# Cache hit: the file already exists — no worker round-trip needed.
|
# Cache hit: the file already exists — no worker round-trip needed.
|
||||||
if asset.status == "ready":
|
if asset.status == "ready":
|
||||||
@@ -212,7 +242,7 @@ def enqueue_edit(
|
|||||||
queue_pos=next_queue_pos(db),
|
queue_pos=next_queue_pos(db),
|
||||||
)
|
)
|
||||||
db.add(job)
|
db.add(job)
|
||||||
asset.ref_count = (asset.ref_count or 0) + 1
|
incr_ref_count(db, asset.id)
|
||||||
|
|
||||||
if asset.status == "ready": # identical clip already produced for this user → instant
|
if asset.status == "ready": # identical clip already produced for this user → instant
|
||||||
job.status = "done"
|
job.status = "done"
|
||||||
@@ -273,8 +303,83 @@ def release_asset(db: Session, job: DownloadJob) -> None:
|
|||||||
asset = db.get(MediaAsset, job.asset_id)
|
asset = db.get(MediaAsset, job.asset_id)
|
||||||
if asset is None:
|
if asset is None:
|
||||||
return
|
return
|
||||||
asset.ref_count = max(0, (asset.ref_count or 0) - 1)
|
new_count = decr_ref_count(db, asset.id)
|
||||||
if asset.ref_count == 0 and asset.status == "ready":
|
if new_count == 0 and asset.status == "ready":
|
||||||
if asset.rel_path:
|
if asset.rel_path:
|
||||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||||
db.delete(asset)
|
db.delete(asset)
|
||||||
|
|
||||||
|
|
||||||
|
# --- Job state transitions ---------------------------------------------------------------------
|
||||||
|
# The single home for the pause/resume/cancel/delete/redownload transitions + their asset
|
||||||
|
# bookkeeping (release, failed-asset reset, fresh-asset rebuild), so the routes are thin callers and
|
||||||
|
# the worker/GC could share the same definitions (B-sub-8). Each commits its own transition; the
|
||||||
|
# route keeps only auth + HTTP mapping + serialization.
|
||||||
|
|
||||||
|
|
||||||
|
def pause(db: Session, job: DownloadJob) -> None:
|
||||||
|
"""Pause a queued/running job (a running worker aborts cooperatively via the cancel poll); a no-op
|
||||||
|
in any other state."""
|
||||||
|
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING):
|
||||||
|
job.status = JobStatus.PAUSED
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def resume(db: Session, job: DownloadJob) -> None:
|
||||||
|
"""Requeue a paused/errored job for a fresh attempt, clearing its error. If the SHARED asset itself
|
||||||
|
failed, reset it to pending so the worker actually re-downloads — otherwise the requeued job would
|
||||||
|
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset)."""
|
||||||
|
if job.status in (JobStatus.PAUSED, JobStatus.ERROR):
|
||||||
|
job.progress = 0
|
||||||
|
job.error = None
|
||||||
|
job.error_code = None
|
||||||
|
if job.asset_id:
|
||||||
|
asset = db.get(MediaAsset, job.asset_id)
|
||||||
|
if asset and asset.status == "error": # MediaAsset.status (no enum), not a JobStatus
|
||||||
|
asset.status = "pending"
|
||||||
|
asset.error = None
|
||||||
|
job.status = JobStatus.QUEUED
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def cancel(db: Session, job: DownloadJob) -> None:
|
||||||
|
"""Cancel a job unless it's already done/canceled, releasing its asset hold WHILE it still counts
|
||||||
|
as holding — `canceled` ∉ HOLDING, so a later delete won't double-release."""
|
||||||
|
if job.status not in (JobStatus.DONE, JobStatus.CANCELED):
|
||||||
|
release_asset(db, job) # release while the job still counts as holding
|
||||||
|
job.status = JobStatus.CANCELED
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def delete(db: Session, job: DownloadJob) -> None:
|
||||||
|
"""Delete a job, releasing its asset hold first (which frees the file iff this was the last holder)."""
|
||||||
|
release_asset(db, job)
|
||||||
|
db.delete(job)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def redownload(db: Session, job: DownloadJob) -> None:
|
||||||
|
"""Rebuild a finished/failed job's file into a FRESH, private asset (the cache sig is salted with
|
||||||
|
the job id + current asset id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||||
|
so any public share link (DownloadLink -> job_id) keeps resolving to the newly-downloaded file.
|
||||||
|
The old hold is dropped first (freeing the old file iff this job was its last holder). The caller
|
||||||
|
has already rejected edit clips + still-in-progress jobs (HTTP 409)."""
|
||||||
|
spec = formats.normalize(job.profile_snapshot)
|
||||||
|
sig = formats.format_sig(spec, salt=f"{job.id}:{job.asset_id}")
|
||||||
|
asset = get_or_create_asset(db, job.source_kind, job.source_ref, sig)
|
||||||
|
populate_from_catalog(db, asset)
|
||||||
|
# Drop the old hold BEFORE re-pointing the job — release_asset keys off the job's current asset.
|
||||||
|
release_asset(db, job)
|
||||||
|
incr_ref_count(db, asset.id)
|
||||||
|
job.asset_id = asset.id
|
||||||
|
job.status = JobStatus.QUEUED
|
||||||
|
job.progress = 0
|
||||||
|
job.phase = None
|
||||||
|
job.error = None
|
||||||
|
job.speed_bps = None
|
||||||
|
job.eta_s = None
|
||||||
|
# Re-derive the display name from the freshly cleaned title on completion (else an older, un-cleaned
|
||||||
|
# name would survive the rebuild).
|
||||||
|
job.display_name = None
|
||||||
|
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
|
||||||
|
db.commit()
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ Plus a Kodi/Plex-readable `<basename>.nfo` (episodedetails) and `<basename>.jpg`
|
|||||||
a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode,
|
a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode,
|
||||||
date-based — the community-standard mapping for YouTube-in-Plex.
|
date-based — the community-standard mapping for YouTube-in-Plex.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import unicodedata
|
import unicodedata
|
||||||
@@ -36,9 +37,27 @@ _DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"}
|
|||||||
# that accented names show in non-UTF-8 tools.
|
# that accented names show in non-UTF-8 tools.
|
||||||
_FOLD_MAP = str.maketrans(
|
_FOLD_MAP = str.maketrans(
|
||||||
{
|
{
|
||||||
"ß": "ss", "ẞ": "SS", "ø": "o", "Ø": "O", "ł": "l", "Ł": "L", "đ": "d", "Đ": "D",
|
"ß": "ss",
|
||||||
"ð": "d", "Ð": "D", "þ": "th", "Þ": "Th", "æ": "ae", "Æ": "AE", "œ": "oe", "Œ": "OE",
|
"ẞ": "SS",
|
||||||
"ı": "i", "İ": "I", "·": " ", "–": "-", "—": "-",
|
"ø": "o",
|
||||||
|
"Ø": "O",
|
||||||
|
"ł": "l",
|
||||||
|
"Ł": "L",
|
||||||
|
"đ": "d",
|
||||||
|
"Đ": "D",
|
||||||
|
"ð": "d",
|
||||||
|
"Ð": "D",
|
||||||
|
"þ": "th",
|
||||||
|
"Þ": "Th",
|
||||||
|
"æ": "ae",
|
||||||
|
"Æ": "AE",
|
||||||
|
"œ": "oe",
|
||||||
|
"Œ": "OE",
|
||||||
|
"ı": "i",
|
||||||
|
"İ": "I",
|
||||||
|
"·": " ",
|
||||||
|
"–": "-",
|
||||||
|
"—": "-",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -174,7 +193,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
|
|||||||
f" <premiered>{aired}</premiered>",
|
f" <premiered>{aired}</premiered>",
|
||||||
f" <year>{year}</year>",
|
f" <year>{year}</year>",
|
||||||
f" <plot>{escape(meta.description or '')}</plot>",
|
f" <plot>{escape(meta.description or '')}</plot>",
|
||||||
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
f' <uniqueid type="{escape(meta.uniqueid_type)}" default="true">{escape(meta.video_id)}</uniqueid>',
|
||||||
]
|
]
|
||||||
if meta.duration_s:
|
if meta.duration_s:
|
||||||
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
||||||
@@ -190,9 +209,7 @@ def _sidecar(media: Path, ext: str) -> Path:
|
|||||||
return media.with_suffix(ext)
|
return media.with_suffix(ext)
|
||||||
|
|
||||||
|
|
||||||
def write_sidecars(
|
def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
|
||||||
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
|
|
||||||
) -> bool:
|
|
||||||
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
||||||
media = abs_path(download_root, rel)
|
media = abs_path(download_root, rel)
|
||||||
try:
|
try:
|
||||||
|
|||||||
+6
-13
@@ -4,6 +4,7 @@ Gmail SMTP + App Password by default. Deliberately fail-soft: if SMTP isn't conf
|
|||||||
or a send errors, we log and return False so the caller can fall back to in-app
|
or a send errors, we log and return False so the caller can fall back to in-app
|
||||||
notifications — email is never load-bearing for the app's core flow.
|
notifications — email is never load-bearing for the app's core flow.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import smtplib
|
import smtplib
|
||||||
import ssl
|
import ssl
|
||||||
@@ -77,8 +78,7 @@ def send_access_approved(email: str) -> bool:
|
|||||||
"Open Siftlode and sign in, and your YouTube subscriptions feed will be ready:\n\n"
|
"Open Siftlode and sign in, and your YouTube subscriptions feed will be ready:\n\n"
|
||||||
f"{settings.app_base}\n\n"
|
f"{settings.app_base}\n\n"
|
||||||
"If you weren't expecting this, you can ignore the message.\n\n"
|
"If you weren't expecting this, you can ignore the message.\n\n"
|
||||||
"— Siftlode"
|
"— Siftlode" + (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
|
||||||
+ (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
|
|
||||||
)
|
)
|
||||||
return _send([email], "You're in — your Siftlode access is approved", body, reply_to=admin)
|
return _send([email], "You're in — your Siftlode access is approved", body, reply_to=admin)
|
||||||
|
|
||||||
@@ -90,8 +90,7 @@ def send_access_denied(email: str) -> bool:
|
|||||||
"Thanks for your interest in Siftlode. After review, your access request wasn't approved "
|
"Thanks for your interest in Siftlode. After review, your access request wasn't approved "
|
||||||
"this time.\n\n"
|
"this time.\n\n"
|
||||||
"If you think this was a mistake, you're welcome to reply and let us know.\n\n"
|
"If you think this was a mistake, you're welcome to reply and let us know.\n\n"
|
||||||
"— Siftlode"
|
"— Siftlode" + (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
|
||||||
+ (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
|
|
||||||
)
|
)
|
||||||
return _send([email], "About your Siftlode access request", body, reply_to=admin)
|
return _send([email], "About your Siftlode access request", body, reply_to=admin)
|
||||||
|
|
||||||
@@ -113,9 +112,7 @@ def send_account_unsuspended(to: str, operator: str | None) -> bool:
|
|||||||
body = (
|
body = (
|
||||||
"Hi,\n\n"
|
"Hi,\n\n"
|
||||||
"Your Siftlode account has been reinstated — the suspension was lifted and you can sign "
|
"Your Siftlode account has been reinstated — the suspension was lifted and you can sign "
|
||||||
"in again."
|
"in again." + contact + "\n\n— Siftlode"
|
||||||
+ contact
|
|
||||||
+ "\n\n— Siftlode"
|
|
||||||
)
|
)
|
||||||
return _send([to], "Your Siftlode account has been reinstated", body, reply_to=operator)
|
return _send([to], "Your Siftlode account has been reinstated", body, reply_to=operator)
|
||||||
|
|
||||||
@@ -130,9 +127,7 @@ def send_account_deleted(to: str, operator: str | None) -> bool:
|
|||||||
"Hi,\n\n"
|
"Hi,\n\n"
|
||||||
"Your Siftlode account and all associated data — subscriptions, tags, watch history, "
|
"Your Siftlode account and all associated data — subscriptions, tags, watch history, "
|
||||||
"playlists and settings — have been permanently deleted. Nothing is retained.\n\n"
|
"playlists and settings — have been permanently deleted. Nothing is retained.\n\n"
|
||||||
"Thanks for having used Siftlode."
|
"Thanks for having used Siftlode." + contact + "\n\n— Siftlode"
|
||||||
+ contact
|
|
||||||
+ "\n\n— Siftlode"
|
|
||||||
)
|
)
|
||||||
return _send([to], "Your Siftlode account has been deleted", body, reply_to=operator)
|
return _send([to], "Your Siftlode account has been deleted", body, reply_to=operator)
|
||||||
|
|
||||||
@@ -147,9 +142,7 @@ def send_account_suspended(to: str, operator: str | None) -> bool:
|
|||||||
body = (
|
body = (
|
||||||
"Hi,\n\n"
|
"Hi,\n\n"
|
||||||
"A sign-in to your Siftlode account was just attempted, but the account is currently "
|
"A sign-in to your Siftlode account was just attempted, but the account is currently "
|
||||||
"suspended, so access was denied.\n\n"
|
"suspended, so access was denied.\n\n" + contact + "— Siftlode"
|
||||||
+ contact
|
|
||||||
+ "— Siftlode"
|
|
||||||
)
|
)
|
||||||
return _send([to], "Your Siftlode account is suspended", body, reply_to=operator)
|
return _send([to], "Your Siftlode account is suspended", body, reply_to=operator)
|
||||||
|
|
||||||
|
|||||||
+195
-59
@@ -100,12 +100,24 @@ def serialize(row) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def tags_by_category(db: Session, tag_ids: list[int]) -> dict[str, list[int]]:
|
||||||
|
"""Group tag ids by their category (language/topic/other). The single source for the category split
|
||||||
|
shared by the feed's tag WHERE (filtered_query) and the facet/count summary (feed_summary), so the
|
||||||
|
two can't disagree on which selected tag belongs to which category."""
|
||||||
|
if not tag_ids:
|
||||||
|
return {} # skip the pointless `WHERE id IN ()` round-trip on the no-tag-filter hot path
|
||||||
|
out: dict[str, list[int]] = {}
|
||||||
|
for tid, cat in db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids))).all():
|
||||||
|
out.setdefault(cat, []).append(tid)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def filtered_query(
|
def filtered_query(
|
||||||
db: Session,
|
db: Session,
|
||||||
user: User,
|
user: User,
|
||||||
*,
|
*,
|
||||||
tags: list[int],
|
tags: list[int],
|
||||||
tag_mode: str,
|
tag_modes: dict[str, str],
|
||||||
channel_ids: list[str],
|
channel_ids: list[str],
|
||||||
q: str | None,
|
q: str | None,
|
||||||
min_duration: int | None,
|
min_duration: int | None,
|
||||||
@@ -131,11 +143,8 @@ def filtered_query(
|
|||||||
either mode via the VideoState outer join."""
|
either mode via the VideoState outer join."""
|
||||||
state = aliased(VideoState)
|
state = aliased(VideoState)
|
||||||
status_expr = func.coalesce(state.status, "new")
|
status_expr = func.coalesce(state.status, "new")
|
||||||
position_expr = func.coalesce(state.position_seconds, 0)
|
|
||||||
|
|
||||||
query = select(*feed_columns(user, state)).join(
|
query = select(*feed_columns(user, state)).join(Channel, Channel.id == Video.channel_id)
|
||||||
Channel, Channel.id == Video.channel_id
|
|
||||||
)
|
|
||||||
|
|
||||||
if scope == "all":
|
if scope == "all":
|
||||||
# Whole shared catalog; subscription is optional (only for priority sort).
|
# Whole shared catalog; subscription is optional (only for priority sort).
|
||||||
@@ -163,9 +172,14 @@ def filtered_query(
|
|||||||
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
|
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
|
||||||
)
|
)
|
||||||
|
|
||||||
query = query.outerjoin(
|
# The state-gated views (watched/hidden/in_progress) only ever return videos that HAVE a
|
||||||
state, and_(state.video_id == Video.id, state.user_id == user.id)
|
# video_states row, so an INNER join lets the planner drive from the user's own small video_states
|
||||||
)
|
# set (~100 rows) instead of scanning all videos and outer-joining (PERF-16). status is NOT NULL on
|
||||||
|
# a present row (server_default 'new'), so the raw-column filters below match the coalesce form
|
||||||
|
# exactly. 'unwatched'/'all' include row-less 'new' videos, so they keep the outer join.
|
||||||
|
state_required = show in ("watched", "hidden", "in_progress")
|
||||||
|
join_state = query.join if state_required else query.outerjoin
|
||||||
|
query = join_state(state, and_(state.video_id == Video.id, state.user_id == user.id))
|
||||||
|
|
||||||
# Hide videos the maintenance job has flagged as currently unplayable (deleted/private/
|
# Hide videos the maintenance job has flagged as currently unplayable (deleted/private/
|
||||||
# abandoned). They're either deleted after a grace period or unhidden if they recover;
|
# abandoned). They're either deleted after a grace period or unhidden if they recover;
|
||||||
@@ -179,9 +193,7 @@ def filtered_query(
|
|||||||
# asking for scope=all + channel_ids (the channel is in this accessible set).
|
# asking for scope=all + channel_ids (the channel is in this accessible set).
|
||||||
accessible_explore = (
|
accessible_explore = (
|
||||||
select(Subscription.channel_id).where(Subscription.user_id == user.id)
|
select(Subscription.channel_id).where(Subscription.user_id == user.id)
|
||||||
).union(
|
).union(select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id))
|
||||||
select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id)
|
|
||||||
)
|
|
||||||
query = query.where(
|
query = query.where(
|
||||||
or_(
|
or_(
|
||||||
Video.via_explore.is_(False),
|
Video.via_explore.is_(False),
|
||||||
@@ -226,13 +238,9 @@ def filtered_query(
|
|||||||
query = query.where(Video.duration_seconds <= max_duration)
|
query = query.where(Video.duration_seconds <= max_duration)
|
||||||
if max_age_days is not None:
|
if max_age_days is not None:
|
||||||
cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400
|
cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400
|
||||||
query = query.where(
|
query = query.where(Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc))
|
||||||
Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)
|
|
||||||
)
|
|
||||||
if published_after is not None:
|
if published_after is not None:
|
||||||
start = datetime.combine(
|
start = datetime.combine(published_after, datetime.min.time(), tzinfo=timezone.utc)
|
||||||
published_after, datetime.min.time(), tzinfo=timezone.utc
|
|
||||||
)
|
|
||||||
query = query.where(Video.published_at >= start)
|
query = query.where(Video.published_at >= start)
|
||||||
if published_before is not None:
|
if published_before is not None:
|
||||||
end = datetime.combine(
|
end = datetime.combine(
|
||||||
@@ -246,15 +254,25 @@ def filtered_query(
|
|||||||
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort.
|
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort.
|
||||||
rank_expr = None
|
rank_expr = None
|
||||||
if q:
|
if q:
|
||||||
|
# Resolve the channels whose NAME matches up front (there are only a few hundred channels, so
|
||||||
|
# this is a tiny standalone scan) and match videos by channel_id. ORing an unindexed
|
||||||
|
# `unaccent(channels.title) ILIKE '%q%'` straight into the per-video predicate defeated the
|
||||||
|
# search_vector GIN — the planner detoasted every non-matching tsvector to test the `@@`
|
||||||
|
# (~2.6s/search on the prod-sized set); channel_id.in_(...) keeps the GIN as a BitmapOr branch
|
||||||
|
# (PERF-12). Empty match -> false() so the OR collapses to the text match alone.
|
||||||
|
like = func.unaccent(f"%{q}%")
|
||||||
|
matching_channel_ids = (
|
||||||
|
db.execute(select(Channel.id).where(func.unaccent(Channel.title).ilike(like)))
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
channel_match = (
|
||||||
|
Video.channel_id.in_(matching_channel_ids) if matching_channel_ids else false()
|
||||||
|
)
|
||||||
ts_str = to_tsquery_str(q)
|
ts_str = to_tsquery_str(q)
|
||||||
if ts_str:
|
if ts_str:
|
||||||
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
|
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
|
||||||
query = query.where(
|
query = query.where(or_(Video.search_vector.op("@@")(tsq), channel_match))
|
||||||
or_(
|
|
||||||
Video.search_vector.op("@@")(tsq),
|
|
||||||
func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
|
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
|
||||||
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
|
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
|
||||||
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
|
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
|
||||||
@@ -262,21 +280,15 @@ def filtered_query(
|
|||||||
rank_expr = cast(func.ts_rank(Video.search_vector, tsq) * 1000000, BigInteger)
|
rank_expr = cast(func.ts_rank(Video.search_vector, tsq) * 1000000, BigInteger)
|
||||||
else:
|
else:
|
||||||
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
|
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
|
||||||
like = func.unaccent(f"%{q}%")
|
query = query.where(or_(func.unaccent(Video.title).ilike(like), channel_match))
|
||||||
query = query.where(
|
|
||||||
or_(
|
|
||||||
func.unaccent(Video.title).ilike(like),
|
|
||||||
func.unaccent(Channel.title).ilike(like),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if tags:
|
if tags:
|
||||||
# AND across tag categories (e.g. language AND topic narrows), OR within a
|
# AND across tag categories (e.g. language AND topic narrows), OR within a category — unless that
|
||||||
# category; the any/all toggle controls multiple topic tags.
|
# category's mode is "and" (its per-category any/all toggle), which requires a channel to carry
|
||||||
cat_rows = db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tags))).all()
|
# ALL its selected tags. ⚠️ feed_summary.passes() reimplements this exact rule in Python for the
|
||||||
by_category: dict[str, list[int]] = {}
|
# sidebar count/facets — keep the two in lockstep (equivalence-tested by
|
||||||
for tag_id, category in cat_rows:
|
# test_feed_summary_matches_count_plus_facets).
|
||||||
by_category.setdefault(category, []).append(tag_id)
|
by_category = tags_by_category(db, tags)
|
||||||
# Facet counting drops the category being counted so its own chips don't zero each
|
# Facet counting drops the category being counted so its own chips don't zero each
|
||||||
# other out (standard drill-down faceting: a category's count ignores its own
|
# other out (standard drill-down faceting: a category's count ignores its own
|
||||||
# selections but still honours the other categories' filters).
|
# selections but still honours the other categories' filters).
|
||||||
@@ -284,7 +296,7 @@ def filtered_query(
|
|||||||
by_category.pop(exclude_tag_category, None)
|
by_category.pop(exclude_tag_category, None)
|
||||||
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||||
for category, ids in by_category.items():
|
for category, ids in by_category.items():
|
||||||
if category == "topic" and tag_mode == "and" and len(ids) > 1:
|
if tag_modes.get(category, "or") == "and" and len(ids) > 1:
|
||||||
sub = (
|
sub = (
|
||||||
select(ChannelTag.channel_id)
|
select(ChannelTag.channel_id)
|
||||||
.where(ChannelTag.tag_id.in_(ids), visible)
|
.where(ChannelTag.tag_id.in_(ids), visible)
|
||||||
@@ -292,9 +304,7 @@ def filtered_query(
|
|||||||
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids)))
|
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids)))
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
sub = select(ChannelTag.channel_id).where(
|
sub = select(ChannelTag.channel_id).where(ChannelTag.tag_id.in_(ids), visible)
|
||||||
ChannelTag.tag_id.in_(ids), visible
|
|
||||||
)
|
|
||||||
query = query.where(Video.channel_id.in_(sub))
|
query = query.where(Video.channel_id.in_(sub))
|
||||||
|
|
||||||
# Content type: Normal / Shorts / Live·Upcoming as a union of enabled types.
|
# Content type: Normal / Shorts / Live·Upcoming as a union of enabled types.
|
||||||
@@ -315,20 +325,98 @@ def filtered_query(
|
|||||||
if show == "unwatched":
|
if show == "unwatched":
|
||||||
query = query.where(status_expr.notin_(("watched", "hidden")))
|
query = query.where(status_expr.notin_(("watched", "hidden")))
|
||||||
elif show == "in_progress":
|
elif show == "in_progress":
|
||||||
# Started but not finished: a stored resume position, not yet watched/hidden.
|
# Started but not finished: a stored resume position, not yet watched/hidden. INNER-joined
|
||||||
|
# (state_required) so the raw state columns are safe and the planner can use video_states.
|
||||||
query = query.where(
|
query = query.where(
|
||||||
and_(position_expr > 0, status_expr.notin_(("watched", "hidden")))
|
and_(state.position_seconds > 0, state.status.notin_(("watched", "hidden")))
|
||||||
)
|
)
|
||||||
elif show == "watched":
|
elif show == "watched":
|
||||||
query = query.where(status_expr == "watched")
|
query = query.where(state.status == "watched")
|
||||||
elif show == "hidden":
|
elif show == "hidden":
|
||||||
query = query.where(status_expr == "hidden")
|
query = query.where(state.status == "hidden")
|
||||||
else: # all
|
else: # all
|
||||||
query = query.where(status_expr != "hidden")
|
query = query.where(status_expr != "hidden")
|
||||||
|
|
||||||
return query, rank_expr
|
return query, rank_expr
|
||||||
|
|
||||||
|
|
||||||
|
def feed_summary(db: Session, user: User, **params) -> dict:
|
||||||
|
"""Single-pass replacement for /feed/count + 3x /facets (PERF-13): the sidebar's total video count
|
||||||
|
and every tag's distinct-channel facet count, derived in Python from ONE grouped query instead of
|
||||||
|
four full passes over the subscribed set on every filter change.
|
||||||
|
|
||||||
|
The one query is the base filter with the TAG predicate removed, grouped by channel: it yields every
|
||||||
|
channel that has a matching video, with its video count. From that ~few-hundred-row map we compute:
|
||||||
|
- count = sum of the video counts of the channels that pass the SELECTED tag filter;
|
||||||
|
- facets = per tag, the number of distinct channels that carry it AND pass the drill-down filter
|
||||||
|
(the tag's own category dropped, unless it's a conjunctive topic-AND — mirroring the
|
||||||
|
old /facets exclude_tag_category logic exactly).
|
||||||
|
Tag matching is channel-based, identical to filtered_query's tag block (AND across categories, OR
|
||||||
|
within, a category in "and" mode requires ALL its selected tags)."""
|
||||||
|
tag_modes = params.get("tag_modes", {})
|
||||||
|
|
||||||
|
# ONE pass: per-channel video counts for the base filter, tags stripped so every candidate channel
|
||||||
|
# is present to facet over.
|
||||||
|
base, _rank = filtered_query(db, user, **{**params, "tags": []})
|
||||||
|
ch_col = base.with_only_columns(Video.channel_id).subquery()
|
||||||
|
channel_counts: dict[str, int] = dict(
|
||||||
|
db.execute(select(ch_col.c.channel_id, func.count()).group_by(ch_col.c.channel_id)).all()
|
||||||
|
)
|
||||||
|
if not channel_counts:
|
||||||
|
return {"count": 0, "facets": {}}
|
||||||
|
|
||||||
|
# channel -> its visible tag ids, plus each tag's category (for grouping the facet output).
|
||||||
|
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||||
|
channel_tags: dict[str, set[int]] = {}
|
||||||
|
tag_category: dict[int, str] = {}
|
||||||
|
for cid, tid, cat in db.execute(
|
||||||
|
select(ChannelTag.channel_id, ChannelTag.tag_id, Tag.category)
|
||||||
|
.join(Tag, Tag.id == ChannelTag.tag_id)
|
||||||
|
.where(visible, ChannelTag.channel_id.in_(channel_counts.keys()))
|
||||||
|
).all():
|
||||||
|
channel_tags.setdefault(cid, set()).add(tid)
|
||||||
|
tag_category[tid] = cat
|
||||||
|
|
||||||
|
# Selected tags grouped by category (read from Tag via the shared helper, so a selected tag that no
|
||||||
|
# base channel carries still filters everything out for its category — matching the SQL `IN (empty)`
|
||||||
|
# behaviour).
|
||||||
|
selected_by_cat: dict[str, set[int]] = {
|
||||||
|
cat: set(ids) for cat, ids in tags_by_category(db, params.get("tags") or []).items()
|
||||||
|
}
|
||||||
|
|
||||||
|
def passes(ch_tags: set[int], exclude_cat: str | None) -> bool:
|
||||||
|
# The Python mirror of filtered_query's SQL tag block (AND across categories, OR within,
|
||||||
|
# topic-AND requires all selected) — keep in lockstep; see the ⚠️ note there.
|
||||||
|
for cat, sel in selected_by_cat.items():
|
||||||
|
if cat == exclude_cat:
|
||||||
|
continue
|
||||||
|
if tag_modes.get(cat, "or") == "and" and len(sel) > 1:
|
||||||
|
if (
|
||||||
|
not sel <= ch_tags
|
||||||
|
): # conjunctive: channel must carry ALL this category's selected tags
|
||||||
|
return False
|
||||||
|
elif not (sel & ch_tags): # disjunctive: channel must carry ANY selected tag
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
count = sum(
|
||||||
|
n for cid, n in channel_counts.items() if passes(channel_tags.get(cid, set()), None)
|
||||||
|
)
|
||||||
|
|
||||||
|
facets: dict[int, int] = {}
|
||||||
|
for category in ("language", "topic", "other"):
|
||||||
|
# Drop this category's own selections so its chips keep independent counts; a conjunctive
|
||||||
|
# topic-AND keeps them (each remaining chip narrows to channels that ALSO have the selected).
|
||||||
|
exclude = None if tag_modes.get(category, "or") == "and" else category
|
||||||
|
for cid, ch_tags in channel_tags.items():
|
||||||
|
if not passes(ch_tags, exclude):
|
||||||
|
continue
|
||||||
|
for tid in ch_tags:
|
||||||
|
if tag_category.get(tid) == category:
|
||||||
|
facets[tid] = facets.get(tid, 0) + 1
|
||||||
|
return {"count": count, "facets": facets}
|
||||||
|
|
||||||
|
|
||||||
# --- Keyset (cursor) pagination ---------------------------------------------
|
# --- Keyset (cursor) pagination ---------------------------------------------
|
||||||
#
|
#
|
||||||
# Each sort is described as an ordered list of key columns (expression + direction
|
# Each sort is described as an ordered list of key columns (expression + direction
|
||||||
@@ -339,25 +427,58 @@ def filtered_query(
|
|||||||
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
||||||
# (OFFSET would shift and duplicate/skip rows).
|
# (OFFSET would shift and duplicate/skip rows).
|
||||||
def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||||
pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"}
|
pub = {"expr": Video.published_at, "asc": False, "nullable": False, "kind": "dt"}
|
||||||
table: dict[str, list[dict]] = {
|
table: dict[str, list[dict]] = {
|
||||||
"newest": [pub],
|
"newest": [pub],
|
||||||
"oldest": [{**pub, "asc": True}],
|
"oldest": [{**pub, "asc": True}],
|
||||||
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
|
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
|
||||||
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
|
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
|
||||||
"duration_desc": [{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}],
|
"duration_desc": [
|
||||||
"duration_asc": [{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}],
|
{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}
|
||||||
|
],
|
||||||
|
"duration_asc": [
|
||||||
|
{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}
|
||||||
|
],
|
||||||
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
|
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
|
||||||
"title_desc": [{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}],
|
"title_desc": [
|
||||||
"subscribers": [{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}],
|
{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}
|
||||||
"subscribers_asc": [{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}],
|
],
|
||||||
|
"subscribers": [
|
||||||
|
{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}
|
||||||
|
],
|
||||||
|
"subscribers_asc": [
|
||||||
|
{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}
|
||||||
|
],
|
||||||
# Your per-channel priority (set in the channel manager), newest first within a tier.
|
# Your per-channel priority (set in the channel manager), newest first within a tier.
|
||||||
# coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
|
# coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
|
||||||
"priority": [{"expr": func.coalesce(Subscription.priority, 0), "asc": False, "nullable": False, "kind": "num"}, pub],
|
"priority": [
|
||||||
"priority_asc": [{"expr": func.coalesce(Subscription.priority, 0), "asc": True, "nullable": False, "kind": "num"}, pub],
|
{
|
||||||
|
"expr": func.coalesce(Subscription.priority, 0),
|
||||||
|
"asc": False,
|
||||||
|
"nullable": False,
|
||||||
|
"kind": "num",
|
||||||
|
},
|
||||||
|
pub,
|
||||||
|
],
|
||||||
|
"priority_asc": [
|
||||||
|
{
|
||||||
|
"expr": func.coalesce(Subscription.priority, 0),
|
||||||
|
"asc": True,
|
||||||
|
"nullable": False,
|
||||||
|
"kind": "num",
|
||||||
|
},
|
||||||
|
pub,
|
||||||
|
],
|
||||||
# Deterministic per-seed shuffle: the same seed yields the same total order, so
|
# Deterministic per-seed shuffle: the same seed yields the same total order, so
|
||||||
# the cursor stays valid across pages of one shuffled run.
|
# the cursor stays valid across pages of one shuffled run.
|
||||||
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
|
"shuffle": [
|
||||||
|
{
|
||||||
|
"expr": func.md5(func.concat(Video.id, str(seed))),
|
||||||
|
"asc": True,
|
||||||
|
"nullable": False,
|
||||||
|
"kind": "str",
|
||||||
|
}
|
||||||
|
],
|
||||||
}
|
}
|
||||||
# Relevance (search-result ranking) only exists when there's a query to rank against;
|
# Relevance (search-result ranking) only exists when there's a query to rank against;
|
||||||
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
|
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
|
||||||
@@ -383,7 +504,16 @@ def keyset_predicate(keys: list[dict], cursor_id: str):
|
|||||||
|
|
||||||
Lexicographic over the key columns then id, NULL-aware for `nulls_last`: a NULL
|
Lexicographic over the key columns then id, NULL-aware for `nulls_last`: a NULL
|
||||||
sorts after every non-NULL value, so a cursor sitting on a non-NULL value must
|
sorts after every non-NULL value, so a cursor sitting on a non-NULL value must
|
||||||
also pull in the NULL group, and equality treats NULL == NULL."""
|
also pull in the NULL group, and equality treats NULL == NULL.
|
||||||
|
|
||||||
|
Built recursively so that for a NON-nullable leading key we can prepend a non-strict
|
||||||
|
range bound (`col <= v` for desc / `col >= v` for asc) as an AND conjunct — logically
|
||||||
|
a no-op over the OR of strict-after branches, but it lets the planner bound the index
|
||||||
|
scan instead of scanning from the top and filtering every earlier row (PERF-15: deep
|
||||||
|
`newest` pages went 51.8k -> ~0.5k buffers). A NULLABLE key can't take that bound (the
|
||||||
|
NULL group sorts after every value and `col <= v` would drop it), so it keeps the pure
|
||||||
|
NULL-aware OR form."""
|
||||||
|
|
||||||
def eq(k):
|
def eq(k):
|
||||||
return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"]
|
return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"]
|
||||||
|
|
||||||
@@ -393,11 +523,17 @@ def keyset_predicate(keys: list[dict], cursor_id: str):
|
|||||||
cmp = k["expr"] > k["value"] if k["asc"] else k["expr"] < k["value"]
|
cmp = k["expr"] > k["value"] if k["asc"] else k["expr"] < k["value"]
|
||||||
return or_(cmp, k["expr"].is_(None)) if k["nullable"] else cmp
|
return or_(cmp, k["expr"].is_(None)) if k["nullable"] else cmp
|
||||||
|
|
||||||
branches = []
|
def rec(i: int):
|
||||||
for j in range(len(keys)):
|
if i == len(keys):
|
||||||
branches.append(and_(*[eq(keys[i]) for i in range(j)], after(keys[j])))
|
return Video.id > cursor_id # all keys tied -> break by the unique id
|
||||||
branches.append(and_(*[eq(k) for k in keys], Video.id > cursor_id))
|
k = keys[i]
|
||||||
return or_(*branches)
|
tail = or_(after(k), and_(eq(k), rec(i + 1)))
|
||||||
|
if not k["nullable"] and k["value"] is not None:
|
||||||
|
bound = k["expr"] >= k["value"] if k["asc"] else k["expr"] <= k["value"]
|
||||||
|
return and_(bound, tail)
|
||||||
|
return tail
|
||||||
|
|
||||||
|
return rec(0)
|
||||||
|
|
||||||
|
|
||||||
def encode_cursor(keys: list[dict], row) -> str:
|
def encode_cursor(keys: list[dict], row) -> str:
|
||||||
|
|||||||
+2
-6
@@ -12,9 +12,7 @@ from fastapi import FastAPI, HTTPException
|
|||||||
_siftlode_logger = logging.getLogger("siftlode")
|
_siftlode_logger = logging.getLogger("siftlode")
|
||||||
if not _siftlode_logger.handlers:
|
if not _siftlode_logger.handlers:
|
||||||
_handler = logging.StreamHandler(sys.stdout)
|
_handler = logging.StreamHandler(sys.stdout)
|
||||||
_handler.setFormatter(
|
_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s"))
|
||||||
logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s")
|
|
||||||
)
|
|
||||||
_siftlode_logger.addHandler(_handler)
|
_siftlode_logger.addHandler(_handler)
|
||||||
_siftlode_logger.setLevel(logging.INFO)
|
_siftlode_logger.setLevel(logging.INFO)
|
||||||
_siftlode_logger.propagate = False
|
_siftlode_logger.propagate = False
|
||||||
@@ -140,9 +138,7 @@ async def setup_gate(request, call_next):
|
|||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
configured = state.is_configured_cached(db)
|
configured = state.is_configured_cached(db)
|
||||||
if not configured:
|
if not configured:
|
||||||
return JSONResponse(
|
return JSONResponse({"detail": "This instance isn't set up yet."}, status_code=503)
|
||||||
{"detail": "This instance isn't set up yet."}, status_code=503
|
|
||||||
)
|
|
||||||
return await call_next(request)
|
return await call_next(request)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+169
-187
@@ -26,9 +26,7 @@ class TimestampMixin:
|
|||||||
"""Adds a server-defaulted `created_at`. Models that index created_at or order it
|
"""Adds a server-defaulted `created_at`. Models that index created_at or order it
|
||||||
differently (QuotaEvent, Notification, Message) keep their own column on purpose."""
|
differently (QuotaEvent, Notification, Message) keep their own column on purpose."""
|
||||||
|
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
DateTime(timezone=True), server_default=func.now()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class UpdatedAtMixin:
|
class UpdatedAtMixin:
|
||||||
@@ -51,18 +49,14 @@ class User(Base, TimestampMixin):
|
|||||||
password_hash: Mapped[str | None] = mapped_column(String(255))
|
password_hash: Mapped[str | None] = mapped_column(String(255))
|
||||||
# Email ownership proven (Google sign-in is implicitly verified; password registration
|
# Email ownership proven (Google sign-in is implicitly verified; password registration
|
||||||
# verifies via a link). Login requires this AND is_active.
|
# verifies via a link). Login requires this AND is_active.
|
||||||
email_verified: Mapped[bool] = mapped_column(
|
email_verified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
# Admin-approved & enabled. A pending password registration starts inactive; admin
|
# Admin-approved & enabled. A pending password registration starts inactive; admin
|
||||||
# approval activates it. A deactivated account can't log in.
|
# approval activates it. A deactivated account can't log in.
|
||||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
is_active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||||
# Admin-imposed access block, distinct from is_active (approval lifecycle). A suspended
|
# Admin-imposed access block, distinct from is_active (approval lifecycle). A suspended
|
||||||
# account can't sign in by any method and its live sessions are rejected; the user is told
|
# account can't sign in by any method and its live sessions are rejected; the user is told
|
||||||
# why and pointed at the operator's contact. The demo account is never suspendable.
|
# why and pointed at the operator's contact. The demo account is never suspendable.
|
||||||
is_suspended: Mapped[bool] = mapped_column(
|
is_suspended: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
# Server-side session revocation (SA4): a monotonic counter bumped on password reset, password
|
# Server-side session revocation (SA4): a monotonic counter bumped on password reset, password
|
||||||
# change, or "log out everywhere". A signed session cookie records the epoch it was minted under;
|
# change, or "log out everywhere". A signed session cookie records the epoch it was minted under;
|
||||||
# current_user rejects any cookie whose epoch is stale, so a stolen/copied cookie dies on those
|
# current_user rejects any cookie whose epoch is stale, so a stolen/copied cookie dies on those
|
||||||
@@ -95,9 +89,7 @@ class OAuthToken(Base, UpdatedAtMixin):
|
|||||||
__tablename__ = "oauth_tokens"
|
__tablename__ = "oauth_tokens"
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), unique=True)
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), unique=True
|
|
||||||
)
|
|
||||||
# Refresh token encrypted at rest (Fernet). Access token is short-lived and low risk.
|
# Refresh token encrypted at rest (Fernet). Access token is short-lived and low risk.
|
||||||
refresh_token_enc: Mapped[str | None] = mapped_column(String)
|
refresh_token_enc: Mapped[str | None] = mapped_column(String)
|
||||||
access_token: Mapped[str | None] = mapped_column(String)
|
access_token: Mapped[str | None] = mapped_column(String)
|
||||||
@@ -115,9 +107,7 @@ class AuthToken(Base, TimestampMixin):
|
|||||||
__tablename__ = "auth_tokens"
|
__tablename__ = "auth_tokens"
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
kind: Mapped[str] = mapped_column(String(16)) # "verify" | "reset"
|
kind: Mapped[str] = mapped_column(String(16)) # "verify" | "reset"
|
||||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
token_hash: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||||
@@ -133,9 +123,7 @@ class Invite(Base):
|
|||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
||||||
# pending | approved | denied
|
# pending | approved | denied
|
||||||
status: Mapped[str] = mapped_column(
|
status: Mapped[str] = mapped_column(String(16), default="pending", server_default="pending")
|
||||||
String(16), default="pending", server_default="pending"
|
|
||||||
)
|
|
||||||
note: Mapped[str | None] = mapped_column(Text)
|
note: Mapped[str | None] = mapped_column(Text)
|
||||||
requested_at: Mapped[datetime] = mapped_column(
|
requested_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), server_default=func.now()
|
DateTime(timezone=True), server_default=func.now()
|
||||||
@@ -189,9 +177,7 @@ class Channel(Base, TimestampMixin):
|
|||||||
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
||||||
backfill_done: Mapped[bool] = mapped_column(
|
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
# True when this channel only entered the catalog because a live YouTube search
|
# True when this channel only entered the catalog because a live YouTube search
|
||||||
# surfaced one of its videos (nobody subscribes to it / it has no organic upload sync).
|
# surfaced one of its videos (nobody subscribes to it / it has no organic upload sync).
|
||||||
# Lets the Library hide search-only "noise" by default.
|
# Lets the Library hide search-only "noise" by default.
|
||||||
@@ -218,9 +204,7 @@ class Subscription(Base, TimestampMixin):
|
|||||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),)
|
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
ForeignKey("users.id", ondelete="CASCADE")
|
|
||||||
)
|
|
||||||
channel_id: Mapped[str] = mapped_column(
|
channel_id: Mapped[str] = mapped_column(
|
||||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
@@ -231,9 +215,7 @@ class Subscription(Base, TimestampMixin):
|
|||||||
# Per-user opt-in to full-history (deep) backfill of this channel. Default off so a
|
# Per-user opt-in to full-history (deep) backfill of this channel. Default off so a
|
||||||
# new user's unique channels don't trigger a big shared-quota burst; the deep
|
# new user's unique channels don't trigger a big shared-quota burst; the deep
|
||||||
# scheduler only picks up channels at least one user has requested.
|
# scheduler only picks up channels at least one user has requested.
|
||||||
deep_requested: Mapped[bool] = mapped_column(
|
deep_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
subscribed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
subscribed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
user: Mapped["User"] = relationship(back_populates="subscriptions")
|
user: Mapped["User"] = relationship(back_populates="subscriptions")
|
||||||
@@ -251,14 +233,21 @@ class Video(Base, TimestampMixin):
|
|||||||
__tablename__ = "videos"
|
__tablename__ = "videos"
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
||||||
channel_id: Mapped[str] = mapped_column(
|
# No standalone channel_id index: the composite `ix_videos_channel_pub_desc`
|
||||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
# (channel_id, published_at DESC NULLS LAST, id) from migration 0067 leads with channel_id, so it
|
||||||
)
|
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) AND the per-channel feed
|
||||||
|
# sort. Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
||||||
|
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||||
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
||||||
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
||||||
description: Mapped[str | None] = mapped_column(Text)
|
description: Mapped[str | None] = mapped_column(Text)
|
||||||
published_at: Mapped[datetime | None] = mapped_column(
|
# NOT NULL (migration 0068): in reality every video carries a publish date, and a nullable key
|
||||||
DateTime(timezone=True), index=True
|
# forced the feed's keyset predicate into a non-sargable `... OR published_at IS NULL` disjunction
|
||||||
|
# that blocked the ix_videos_published_at_desc range (deep pages scanned from the top and filtered —
|
||||||
|
# PERF-15). server_default now() is the safety net for the rare date-less ingest (the enrich fills
|
||||||
|
# the real date); the ix_videos_published_at index stays for the `oldest` sort.
|
||||||
|
published_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), index=True, nullable=False, server_default=func.now()
|
||||||
)
|
)
|
||||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||||
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||||
@@ -276,27 +265,23 @@ class Video(Base, TimestampMixin):
|
|||||||
# folds YouTube's own relevance judgement into the local search: a video YouTube returned
|
# folds YouTube's own relevance judgement into the local search: a video YouTube returned
|
||||||
# for "ti amo magyarul" becomes findable by that query locally even if its title lacks it.
|
# for "ti amo magyarul" becomes findable by that query locally even if its title lacks it.
|
||||||
search_terms: Mapped[str | None] = mapped_column(Text)
|
search_terms: Mapped[str | None] = mapped_column(Text)
|
||||||
is_short: Mapped[bool] = mapped_column(
|
# No btree here or on the flag columns below: migration 0069 (PERF-3) dropped the never-picked
|
||||||
Boolean, default=False, server_default="false", index=True
|
# boolean btrees and replaced the useful ones with partial indexes over the target subset
|
||||||
)
|
# (via_search=true, NOT shorts_probed, unavailable_since IS NOT NULL) — migration-defined, like the
|
||||||
|
# search_vector GIN.
|
||||||
|
is_short: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
# Whether the youtube.com/shorts/<id> probe has run for this video.
|
# Whether the youtube.com/shorts/<id> probe has run for this video.
|
||||||
shorts_probed: Mapped[bool] = mapped_column(
|
shorts_probed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false", index=True
|
|
||||||
)
|
|
||||||
# True when this video first entered the catalog via a live YouTube search (not an
|
# True when this video first entered the catalog via a live YouTube search (not an
|
||||||
# organic channel/playlist sync). Used to hide search-discovered "noise" from the
|
# organic channel/playlist sync). Used to hide search-discovered "noise" from the
|
||||||
# Library by default; left False by the organic ingest paths.
|
# Library by default; left False by the organic ingest paths.
|
||||||
via_search: Mapped[bool] = mapped_column(
|
via_search: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false", index=True
|
|
||||||
)
|
|
||||||
# True when this video entered the catalog because a user EXPLORED its (un-subscribed)
|
# True when this video entered the catalog because a user EXPLORED its (un-subscribed)
|
||||||
# channel. Like via_search it's hidden from the shared Library by default, but it's
|
# channel. Like via_search it's hidden from the shared Library by default, but it's
|
||||||
# additionally PER-USER private: a via_explore video is only visible to users who
|
# additionally PER-USER private: a via_explore video is only visible to users who
|
||||||
# explored or subscribe to its channel (a gate in the feed query), so one user's
|
# explored or subscribe to its channel (a gate in the feed query), so one user's
|
||||||
# curiosity never floods everyone else's catalog. Left False by the organic ingest paths.
|
# curiosity never floods everyone else's catalog. Left False by the organic ingest paths.
|
||||||
via_explore: Mapped[bool] = mapped_column(
|
via_explore: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false", index=True
|
|
||||||
)
|
|
||||||
# none | live | upcoming | premiere | was_live
|
# none | live | upcoming | premiere | was_live
|
||||||
live_status: Mapped[str] = mapped_column(
|
live_status: Mapped[str] = mapped_column(
|
||||||
String(16), default="none", server_default="none", index=True
|
String(16), default="none", server_default="none", index=True
|
||||||
@@ -306,17 +291,18 @@ class Video(Base, TimestampMixin):
|
|||||||
# maintenance/validation job to decide whether a video is playable anywhere.
|
# maintenance/validation job to decide whether a video is playable anywhere.
|
||||||
embeddable: Mapped[bool | None] = mapped_column(Boolean)
|
embeddable: Mapped[bool | None] = mapped_column(Boolean)
|
||||||
privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private
|
privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private
|
||||||
upload_status: Mapped[str | None] = mapped_column(String(16)) # processed|failed|rejected|deleted
|
upload_status: Mapped[str | None] = mapped_column(
|
||||||
|
String(16)
|
||||||
|
) # processed|failed|rejected|deleted
|
||||||
# Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation
|
# Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation
|
||||||
# (least-recently-checked first); unavailable_since marks a video as currently
|
# (least-recently-checked first); unavailable_since marks a video as currently
|
||||||
# unplayable (hidden from the feed immediately, hard-deleted after a grace period);
|
# unplayable (hidden from the feed immediately, hard-deleted after a grace period);
|
||||||
# unavailable_reason records why (deleted|private|paywalled|abandoned|stuck_live).
|
# unavailable_reason records why (deleted|private|paywalled|abandoned|stuck_live).
|
||||||
last_checked_at: Mapped[datetime | None] = mapped_column(
|
# last_checked_at: no plain btree — migration 0069 (PERF-18) replaced it with the partial
|
||||||
DateTime(timezone=True), index=True
|
# ix_videos_recheck_order (last_checked_at NULLS FIRST WHERE unavailable_since IS NULL) that the
|
||||||
)
|
# revalidate job's exact ORDER BY rides; unavailable_since is a partial WHERE IS NOT NULL (PERF-3).
|
||||||
unavailable_since: Mapped[datetime | None] = mapped_column(
|
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
DateTime(timezone=True), index=True
|
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
)
|
|
||||||
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
|
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
|
||||||
|
|
||||||
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
|
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
|
||||||
@@ -350,9 +336,7 @@ class Tag(Base, TimestampMixin):
|
|||||||
name: Mapped[str] = mapped_column(String(64))
|
name: Mapped[str] = mapped_column(String(64))
|
||||||
color: Mapped[str | None] = mapped_column(String(16))
|
color: Mapped[str | None] = mapped_column(String(16))
|
||||||
# language | topic | other
|
# language | topic | other
|
||||||
category: Mapped[str] = mapped_column(
|
category: Mapped[str] = mapped_column(String(16), default="other", server_default="other")
|
||||||
String(16), default="other", server_default="other"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class ChannelTag(Base):
|
class ChannelTag(Base):
|
||||||
@@ -360,21 +344,15 @@ class ChannelTag(Base):
|
|||||||
regenerated freely; source='user' rows are never touched by the system."""
|
regenerated freely; source='user' rows are never touched by the system."""
|
||||||
|
|
||||||
__tablename__ = "channel_tags"
|
__tablename__ = "channel_tags"
|
||||||
__table_args__ = (
|
__table_args__ = (UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),)
|
||||||
UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
channel_id: Mapped[str] = mapped_column(
|
channel_id: Mapped[str] = mapped_column(
|
||||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
tag_id: Mapped[int] = mapped_column(
|
tag_id: Mapped[int] = mapped_column(ForeignKey("tags.id", ondelete="CASCADE"), index=True)
|
||||||
ForeignKey("tags.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
# NULL = system/auto link visible to everyone; set = a specific user's link.
|
# NULL = system/auto link visible to everyone; set = a specific user's link.
|
||||||
user_id: Mapped[int | None] = mapped_column(
|
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
ForeignKey("users.id", ondelete="CASCADE")
|
|
||||||
)
|
|
||||||
source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
|
source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
|
||||||
confidence: Mapped[float | None] = mapped_column(Float)
|
confidence: Mapped[float | None] = mapped_column(Float)
|
||||||
|
|
||||||
@@ -387,12 +365,8 @@ class VideoState(Base, UpdatedAtMixin):
|
|||||||
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_video"),)
|
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_video"),)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
ForeignKey("users.id", ondelete="CASCADE")
|
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||||
)
|
|
||||||
video_id: Mapped[str] = mapped_column(
|
|
||||||
ForeignKey("videos.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
# new | watched | hidden ("saved" is now Watch-later playlist membership, not a status)
|
# new | watched | hidden ("saved" is now Watch-later playlist membership, not a status)
|
||||||
status: Mapped[str] = mapped_column(String(12), default="new", server_default="new")
|
status: Mapped[str] = mapped_column(String(12), default="new", server_default="new")
|
||||||
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
@@ -417,12 +391,8 @@ class SearchFind(Base, TimestampMixin):
|
|||||||
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
|
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
ForeignKey("users.id", ondelete="CASCADE")
|
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||||
)
|
|
||||||
video_id: Mapped[str] = mapped_column(
|
|
||||||
ForeignKey("videos.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class ExploredChannel(Base, TimestampMixin):
|
class ExploredChannel(Base, TimestampMixin):
|
||||||
@@ -437,14 +407,10 @@ class ExploredChannel(Base, TimestampMixin):
|
|||||||
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
|
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
|
||||||
|
|
||||||
__tablename__ = "explored_channels"
|
__tablename__ = "explored_channels"
|
||||||
__table_args__ = (
|
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),)
|
||||||
UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
ForeignKey("users.id", ondelete="CASCADE")
|
|
||||||
)
|
|
||||||
channel_id: Mapped[str] = mapped_column(
|
channel_id: Mapped[str] = mapped_column(
|
||||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
@@ -456,14 +422,10 @@ class BlockedChannel(Base, TimestampMixin):
|
|||||||
views — a personal "never show me this channel again" that doesn't affect other users."""
|
views — a personal "never show me this channel again" that doesn't affect other users."""
|
||||||
|
|
||||||
__tablename__ = "blocked_channels"
|
__tablename__ = "blocked_channels"
|
||||||
__table_args__ = (
|
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),)
|
||||||
UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
ForeignKey("users.id", ondelete="CASCADE")
|
|
||||||
)
|
|
||||||
channel_id: Mapped[str] = mapped_column(
|
channel_id: Mapped[str] = mapped_column(
|
||||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
@@ -528,19 +490,12 @@ class AppState(Base):
|
|||||||
__tablename__ = "app_state"
|
__tablename__ = "app_state"
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True, default=1)
|
id: Mapped[int] = mapped_column(primary_key=True, default=1)
|
||||||
sync_paused: Mapped[bool] = mapped_column(
|
sync_paused: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
# Admin override for the maintenance job's rolling re-validation batch (videos re-checked
|
|
||||||
# per run). NULL = use the config/env default (settings.maintenance_revalidate_batch).
|
|
||||||
maintenance_revalidate_batch: Mapped[int | None] = mapped_column(Integer)
|
|
||||||
# First-run install wizard. `configured` is False on a fresh instance (the app runs in
|
# First-run install wizard. `configured` is False on a fresh instance (the app runs in
|
||||||
# setup mode: only the wizard works) and flipped True when the wizard finishes. While
|
# setup mode: only the wizard works) and flipped True when the wizard finishes. While
|
||||||
# unconfigured, a one-time setup token (only its SHA-256 hash stored) gates the wizard;
|
# unconfigured, a one-time setup token (only its SHA-256 hash stored) gates the wizard;
|
||||||
# the raw token + URL are printed to the container logs on each startup.
|
# the raw token + URL are printed to the container logs on each startup.
|
||||||
configured: Mapped[bool] = mapped_column(
|
configured: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
setup_token_hash: Mapped[str | None] = mapped_column(String(64))
|
setup_token_hash: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
|
||||||
|
|
||||||
@@ -586,14 +541,10 @@ class Playlist(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
__tablename__ = "playlists"
|
__tablename__ = "playlists"
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
name: Mapped[str] = mapped_column(String(255))
|
name: Mapped[str] = mapped_column(String(255))
|
||||||
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||||
source: Mapped[str] = mapped_column(
|
source: Mapped[str] = mapped_column(String(16), default="local", server_default="local")
|
||||||
String(16), default="local", server_default="local"
|
|
||||||
)
|
|
||||||
yt_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
yt_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
||||||
# True when the current items/name differ from `synced_fingerprint` (unpushed edits).
|
# True when the current items/name differ from `synced_fingerprint` (unpushed edits).
|
||||||
dirty: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
dirty: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
@@ -613,45 +564,37 @@ class PlaylistItem(Base):
|
|||||||
"""A video's membership in a playlist, with its order within that playlist."""
|
"""A video's membership in a playlist, with its order within that playlist."""
|
||||||
|
|
||||||
__tablename__ = "playlist_items"
|
__tablename__ = "playlist_items"
|
||||||
__table_args__ = (
|
__table_args__ = (UniqueConstraint("playlist_id", "video_id", name="uq_playlist_video"),)
|
||||||
UniqueConstraint("playlist_id", "video_id", name="uq_playlist_video"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
playlist_id: Mapped[int] = mapped_column(
|
playlist_id: Mapped[int] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
|
||||||
ForeignKey("playlists.id", ondelete="CASCADE")
|
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||||
)
|
|
||||||
video_id: Mapped[str] = mapped_column(
|
|
||||||
ForeignKey("videos.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
added_at: Mapped[datetime] = mapped_column(
|
added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
DateTime(timezone=True), server_default=func.now()
|
|
||||||
)
|
|
||||||
|
|
||||||
playlist: Mapped["Playlist"] = relationship(back_populates="items")
|
playlist: Mapped["Playlist"] = relationship(back_populates="items")
|
||||||
|
|
||||||
|
|
||||||
class SavedView(Base, TimestampMixin, UpdatedAtMixin):
|
class SavedView(Base, TimestampMixin, UpdatedAtMixin):
|
||||||
"""A per-user named snapshot of the feed's filter/sort state (a "smart view").
|
"""A per-user named snapshot of a module's filter/sort state (a "smart view").
|
||||||
|
|
||||||
`filters` is the serialized FeedFilters JSON the frontend applies with one click.
|
`source` names the module the view belongs to ("feed" | "plex"); `filters` is that module's
|
||||||
`position` orders them in the sidebar (drag-reorder); `is_default` marks the one view
|
serialized filter JSON the frontend applies with one click. `position` orders them in the sidebar
|
||||||
applied automatically on load — at most one per user, enforced by the API on set."""
|
(drag-reorder, per source); `is_default` marks the one view applied automatically on load — at most
|
||||||
|
one per (user, source), enforced by the API on set. The name is unique per (user, source)."""
|
||||||
|
|
||||||
__tablename__ = "saved_views"
|
__tablename__ = "saved_views"
|
||||||
__table_args__ = (UniqueConstraint("user_id", "name", name="uq_user_saved_view"),)
|
__table_args__ = (
|
||||||
|
UniqueConstraint("user_id", "source", "name", name="uq_user_source_saved_view"),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
ForeignKey("users.id", ondelete="CASCADE")
|
source: Mapped[str] = mapped_column(String(16), default="feed", server_default="feed")
|
||||||
)
|
|
||||||
name: Mapped[str] = mapped_column(String(80))
|
name: Mapped[str] = mapped_column(String(80))
|
||||||
filters: Mapped[dict] = mapped_column(JSON)
|
filters: Mapped[dict] = mapped_column(JSON)
|
||||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
is_default: Mapped[bool] = mapped_column(
|
is_default: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Notification(Base):
|
class Notification(Base):
|
||||||
@@ -669,19 +612,13 @@ class Notification(Base):
|
|||||||
__tablename__ = "notifications"
|
__tablename__ = "notifications"
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
type: Mapped[str] = mapped_column(String(32), index=True) # e.g. "maintenance"
|
type: Mapped[str] = mapped_column(String(32), index=True) # e.g. "maintenance"
|
||||||
title: Mapped[str] = mapped_column(String(255))
|
title: Mapped[str] = mapped_column(String(255))
|
||||||
body: Mapped[str | None] = mapped_column(Text)
|
body: Mapped[str | None] = mapped_column(Text)
|
||||||
data: Mapped[dict | None] = mapped_column(JSON)
|
data: Mapped[dict | None] = mapped_column(JSON)
|
||||||
read: Mapped[bool] = mapped_column(
|
read: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false", index=True)
|
||||||
Boolean, default=False, server_default="false", index=True
|
dismissed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
)
|
|
||||||
dismissed: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), server_default=func.now(), index=True
|
DateTime(timezone=True), server_default=func.now(), index=True
|
||||||
)
|
)
|
||||||
@@ -707,9 +644,7 @@ class Message(Base):
|
|||||||
__tablename__ = "messages"
|
__tablename__ = "messages"
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
kind: Mapped[str] = mapped_column(
|
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||||
String(16), default="user", server_default="user"
|
|
||||||
)
|
|
||||||
# NULL for system messages (no human sender).
|
# NULL for system messages (no human sender).
|
||||||
sender_id: Mapped[int | None] = mapped_column(
|
sender_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||||
@@ -769,13 +704,9 @@ class MediaAsset(Base, TimestampMixin):
|
|||||||
container: Mapped[str | None] = mapped_column(String(16))
|
container: Mapped[str | None] = mapped_column(String(16))
|
||||||
vcodec: Mapped[str | None] = mapped_column(String(32))
|
vcodec: Mapped[str | None] = mapped_column(String(32))
|
||||||
acodec: Mapped[str | None] = mapped_column(String(32))
|
acodec: Mapped[str | None] = mapped_column(String(32))
|
||||||
nfo_written: Mapped[bool] = mapped_column(
|
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
# Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
|
# Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
|
||||||
gc_notified: Mapped[bool] = mapped_column(
|
gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||||
title: Mapped[str | None] = mapped_column(Text)
|
title: Mapped[str | None] = mapped_column(Text)
|
||||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||||
@@ -811,9 +742,7 @@ class DownloadProfile(Base, TimestampMixin):
|
|||||||
) # NULL for a builtin preset
|
) # NULL for a builtin preset
|
||||||
name: Mapped[str] = mapped_column(String(80))
|
name: Mapped[str] = mapped_column(String(80))
|
||||||
spec: Mapped[dict] = mapped_column(JSON)
|
spec: Mapped[dict] = mapped_column(JSON)
|
||||||
is_builtin: Mapped[bool] = mapped_column(
|
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
|
||||||
|
|
||||||
@@ -838,18 +767,14 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
__table_args__ = (Index("ix_download_jobs_claim", "status", "queue_pos"),)
|
__table_args__ = (Index("ix_download_jobs_claim", "status", "queue_pos"),)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
asset_id: Mapped[int | None] = mapped_column(
|
asset_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
|
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
|
||||||
)
|
)
|
||||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||||
source_ref: Mapped[str] = mapped_column(String(512))
|
source_ref: Mapped[str] = mapped_column(String(512))
|
||||||
# "download" (default) or "edit" (a per-user trim/crop derivative of another job).
|
# "download" (default) or "edit" (a per-user trim/crop derivative of another job).
|
||||||
job_kind: Mapped[str] = mapped_column(
|
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
||||||
String(16), default="download", server_default="download"
|
|
||||||
)
|
|
||||||
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
||||||
source_job_id: Mapped[int | None] = mapped_column(
|
source_job_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("download_jobs.id", ondelete="SET NULL")
|
ForeignKey("download_jobs.id", ondelete="SET NULL")
|
||||||
@@ -878,7 +803,9 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
eta_s: Mapped[int | None] = mapped_column(Integer)
|
eta_s: Mapped[int | None] = mapped_column(Integer)
|
||||||
phase: Mapped[str | None] = mapped_column(String(32))
|
phase: Mapped[str | None] = mapped_column(String(32))
|
||||||
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
|
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
|
||||||
error_code: Mapped[str | None] = mapped_column(String(24)) # category, see downloads.errors.classify
|
error_code: Mapped[str | None] = mapped_column(
|
||||||
|
String(24)
|
||||||
|
) # category, see downloads.errors.classify
|
||||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
|
||||||
|
|
||||||
@@ -887,20 +814,14 @@ class DownloadShare(Base, TimestampMixin):
|
|||||||
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
|
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
|
||||||
|
|
||||||
__tablename__ = "download_shares"
|
__tablename__ = "download_shares"
|
||||||
__table_args__ = (
|
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
|
||||||
UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
job_id: Mapped[int] = mapped_column(
|
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
|
||||||
ForeignKey("download_jobs.id", ondelete="CASCADE")
|
|
||||||
)
|
|
||||||
from_user_id: Mapped[int] = mapped_column(
|
from_user_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
to_user_id: Mapped[int] = mapped_column(
|
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadQuota(Base, UpdatedAtMixin):
|
class DownloadQuota(Base, UpdatedAtMixin):
|
||||||
@@ -916,9 +837,7 @@ class DownloadQuota(Base, UpdatedAtMixin):
|
|||||||
max_bytes: Mapped[int] = mapped_column(BigInteger)
|
max_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||||
max_concurrent: Mapped[int] = mapped_column(Integer)
|
max_concurrent: Mapped[int] = mapped_column(Integer)
|
||||||
max_jobs: Mapped[int] = mapped_column(Integer)
|
max_jobs: Mapped[int] = mapped_column(Integer)
|
||||||
unlimited: Mapped[bool] = mapped_column(
|
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadLink(Base, TimestampMixin):
|
class DownloadLink(Base, TimestampMixin):
|
||||||
@@ -937,12 +856,8 @@ class DownloadLink(Base, TimestampMixin):
|
|||||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
token: Mapped[str] = mapped_column(String(64), unique=True)
|
token: Mapped[str] = mapped_column(String(64), unique=True)
|
||||||
created_by: Mapped[int | None] = mapped_column(
|
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
ForeignKey("users.id", ondelete="CASCADE")
|
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
)
|
|
||||||
allow_download: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
password_hash: Mapped[str | None] = mapped_column(Text)
|
password_hash: Mapped[str | None] = mapped_column(Text)
|
||||||
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
|
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
|
||||||
# password immediately invalidates any grants issued under the old one (instead of them living
|
# password immediately invalidates any grants issued under the old one (instead of them living
|
||||||
@@ -1029,10 +944,14 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
title: Mapped[str] = mapped_column(String(512))
|
title: Mapped[str] = mapped_column(String(512))
|
||||||
summary: Mapped[str | None] = mapped_column(Text)
|
summary: Mapped[str | None] = mapped_column(Text)
|
||||||
year: Mapped[int | None] = mapped_column(Integer)
|
year: Mapped[int | None] = mapped_column(Integer)
|
||||||
thumb_key: Mapped[str | None] = mapped_column(String(512)) # Plex thumb path (proxied, not stored)
|
thumb_key: Mapped[str | None] = mapped_column(
|
||||||
|
String(512)
|
||||||
|
) # Plex thumb path (proxied, not stored)
|
||||||
art_key: Mapped[str | None] = mapped_column(String(512))
|
art_key: Mapped[str | None] = mapped_column(String(512))
|
||||||
child_count: Mapped[int | None] = mapped_column(Integer) # seasons
|
child_count: Mapped[int | None] = mapped_column(Integer) # seasons
|
||||||
collection_keys: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # Plex collections this show is in
|
collection_keys: Mapped[list | None] = mapped_column(
|
||||||
|
JSONB(none_as_null=True)
|
||||||
|
) # Plex collections this show is in
|
||||||
added_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
added_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||||
# Filterable / orderable metadata mirrored from the show section listing (parallels PlexItem).
|
# Filterable / orderable metadata mirrored from the show section listing (parallels PlexItem).
|
||||||
rating: Mapped[float | None] = mapped_column(Float, index=True) # Plex audienceRating (~IMDb)
|
rating: Mapped[float | None] = mapped_column(Float, index=True) # Plex audienceRating (~IMDb)
|
||||||
@@ -1045,15 +964,39 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
genres: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
genres: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
directors: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
directors: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
cast_names: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
cast_names: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
people_text: Mapped[str | None] = mapped_column(Text) # cast+directors, folded into search_vector
|
people_text: Mapped[str | None] = mapped_column(
|
||||||
|
Text
|
||||||
|
) # cast+directors, folded into search_vector
|
||||||
# Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
# Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
||||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
||||||
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
||||||
imdb_rating: Mapped[float | None] = mapped_column(Float) # IMDb-specific score (vs `rating` = audienceRating)
|
imdb_rating: Mapped[float | None] = mapped_column(
|
||||||
|
Float
|
||||||
|
) # IMDb-specific score (vs `rating` = audienceRating)
|
||||||
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
|
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
|
||||||
tagline: Mapped[str | None] = mapped_column(Text)
|
tagline: Mapped[str | None] = mapped_column(Text)
|
||||||
full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}]
|
full_cast: Mapped[list | None] = mapped_column(
|
||||||
|
JSONB(none_as_null=True)
|
||||||
|
) # ordered [{name, role, thumb}]
|
||||||
|
# --- TMDB enrichment (E5 S2): see the identical block on plex_items for the full rationale. Additive;
|
||||||
|
# served from the row; NULL until enriched. (`budget`/`revenue` stay NULL for shows — TV has none.) ---
|
||||||
|
tmdb_id: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
tmdb_rating: Mapped[float | None] = mapped_column(Float) # TMDb vote_average (0–10)
|
||||||
|
tmdb_vote_count: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
budget: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
|
revenue: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
|
keywords: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||||
|
crew: Mapped[list | None] = mapped_column(
|
||||||
|
JSONB(none_as_null=True)
|
||||||
|
) # [{name, job, tmdb_id, thumb}]
|
||||||
|
studios: Mapped[list | None] = mapped_column(
|
||||||
|
JSONB(none_as_null=True)
|
||||||
|
) # TMDB production companies [str]
|
||||||
|
production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||||
|
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
|
backdrops: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
||||||
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
||||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||||
@@ -1167,10 +1110,38 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
||||||
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
||||||
imdb_rating: Mapped[float | None] = mapped_column(Float) # IMDb-specific score (vs `rating` = audienceRating)
|
imdb_rating: Mapped[float | None] = mapped_column(
|
||||||
|
Float
|
||||||
|
) # IMDb-specific score (vs `rating` = audienceRating)
|
||||||
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
|
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
|
||||||
tagline: Mapped[str | None] = mapped_column(Text)
|
tagline: Mapped[str | None] = mapped_column(Text)
|
||||||
full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}]
|
full_cast: Mapped[list | None] = mapped_column(
|
||||||
|
JSONB(none_as_null=True)
|
||||||
|
) # ordered [{name, role, thumb}]
|
||||||
|
# --- TMDB enrichment (E5 S2): matched by the canonical `guid` ("tmdb:12345"), filled by the TMDB
|
||||||
|
# backfill job, SERVED FROM THIS ROW (never a live TMDB call). Additive — imdb_rating/rating
|
||||||
|
# above stay Plex-sourced; `tmdb_rating` is the distinct TMDb community score. NULL until enriched;
|
||||||
|
# `tmdb_enriched_at` is the "tried" watermark (stamped on a definitive no-match too, so an
|
||||||
|
# unresolvable title isn't re-fetched every run; left NULL only on a transient fetch error). ---
|
||||||
|
tmdb_id: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
tmdb_rating: Mapped[float | None] = mapped_column(Float) # TMDb vote_average (0–10)
|
||||||
|
tmdb_vote_count: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
budget: Mapped[int | None] = mapped_column(BigInteger) # USD (movies; TV has none)
|
||||||
|
revenue: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
|
keywords: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||||
|
crew: Mapped[list | None] = mapped_column(
|
||||||
|
JSONB(none_as_null=True)
|
||||||
|
) # [{name, job, tmdb_id, thumb}]
|
||||||
|
studios: Mapped[list | None] = mapped_column(
|
||||||
|
JSONB(none_as_null=True)
|
||||||
|
) # TMDB production companies [str]
|
||||||
|
production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||||
|
# [{tmdb_id, title, poster, media_type}] — resolved to library rows (by guid) at serve time.
|
||||||
|
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
|
backdrops: Mapped[list | None] = mapped_column(
|
||||||
|
JSONB(none_as_null=True)
|
||||||
|
) # TMDB backdrop file_paths [str] (crossfade pool, S8)
|
||||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
||||||
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
||||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||||
@@ -1211,6 +1182,23 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
name: Mapped[str] = mapped_column(String(255), unique=True, index=True)
|
name: Mapped[str] = mapped_column(String(255), unique=True, index=True)
|
||||||
photo_thumb: Mapped[str | None] = mapped_column(Text) # proxied /person-image URL, or None
|
photo_thumb: Mapped[str | None] = mapped_column(Text) # proxied /person-image URL, or None
|
||||||
|
# TMDB person id, linked by name from a title's TMDB credits (E5 S2 lays the foundation; the /person
|
||||||
|
# biography fetch + person-page render land in S6). NULL until a TMDB-enriched title names this person.
|
||||||
|
tmdb_person_id: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
# --- TMDB person enrichment (E5 S6): fetched from /person/{tmdb_person_id} the first time the person
|
||||||
|
# page is opened, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||||
|
# no-op fetch so it isn't retried every open). `known_for` = top combined-credits [{tmdb_id,
|
||||||
|
# media_type, title}], resolved to library rows by guid at serve time (like recommendations). ---
|
||||||
|
biography: Mapped[str | None] = mapped_column(Text)
|
||||||
|
birthday: Mapped[str | None] = mapped_column(String(10)) # "YYYY-MM-DD", served as-is
|
||||||
|
deathday: Mapped[str | None] = mapped_column(String(10))
|
||||||
|
place_of_birth: Mapped[str | None] = mapped_column(Text)
|
||||||
|
known_for_department: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
also_known_as: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||||
|
known_for: Mapped[list | None] = mapped_column(
|
||||||
|
JSONB(none_as_null=True)
|
||||||
|
) # [{tmdb_id, media_type, title}]
|
||||||
|
tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
|
||||||
class PlexCollection(Base, TimestampMixin, UpdatedAtMixin):
|
class PlexCollection(Base, TimestampMixin, UpdatedAtMixin):
|
||||||
@@ -1245,9 +1233,7 @@ class PlexState(Base, UpdatedAtMixin):
|
|||||||
__table_args__ = (UniqueConstraint("user_id", "item_id", name="uq_plex_user_item"),)
|
__table_args__ = (UniqueConstraint("user_id", "item_id", name="uq_plex_user_item"),)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
ForeignKey("users.id", ondelete="CASCADE")
|
|
||||||
)
|
|
||||||
item_id: Mapped[int] = mapped_column(
|
item_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
|
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
@@ -1257,9 +1243,7 @@ class PlexState(Base, UpdatedAtMixin):
|
|||||||
)
|
)
|
||||||
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
synced_to_plex: Mapped[bool] = mapped_column(
|
synced_to_plex: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
Boolean, default=False, server_default="false"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class PlexLink(Base, UpdatedAtMixin):
|
class PlexLink(Base, UpdatedAtMixin):
|
||||||
@@ -1314,9 +1298,7 @@ class PlexPlaylistItem(Base):
|
|||||||
__table_args__ = (UniqueConstraint("playlist_id", "item_id", name="uq_plex_playlist_item"),)
|
__table_args__ = (UniqueConstraint("playlist_id", "item_id", name="uq_plex_playlist_item"),)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
playlist_id: Mapped[int] = mapped_column(
|
playlist_id: Mapped[int] = mapped_column(ForeignKey("plex_playlists.id", ondelete="CASCADE"))
|
||||||
ForeignKey("plex_playlists.id", ondelete="CASCADE")
|
|
||||||
)
|
|
||||||
item_id: Mapped[int] = mapped_column(
|
item_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
|
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user