Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3682e6a01f | ||
|
|
aca355f43b | ||
|
|
f41c81eb56 | ||
|
|
614c5528ad | ||
|
|
104bd1e3a6 | ||
|
|
584e9ced0c | ||
|
|
b4ad59274b | ||
|
|
fe0b34cd00 | ||
|
|
9eaa01617c | ||
|
|
fdca12968b | ||
|
|
94fcc40bde | ||
|
|
fb2c843fa9 | ||
|
|
bb6cb6e335 | ||
|
|
bdbff89507 | ||
|
|
9b87838e3a | ||
|
|
f014416e31 | ||
|
|
3369587412 | ||
|
|
9af4e8f4ff | ||
|
|
a0b725d779 | ||
|
|
bc33b2ed16 | ||
|
|
5280949667 | ||
|
|
4892c38466 | ||
|
|
9ae6c47d6c | ||
|
|
b7a6451dba | ||
|
|
11e32163b8 | ||
|
|
257dc5741d | ||
|
|
f52bc5898b | ||
|
|
d0033617e2 | ||
|
|
f80ed115ad | ||
|
|
7fbb996297 | ||
|
|
9cd4ae09d4 | ||
|
|
00527d0cf3 | ||
|
|
8dc4976c2c | ||
|
|
56a2ecdec3 | ||
|
|
8299ea72b7 | ||
|
|
6c918d1dff | ||
|
|
3f8257059d | ||
|
|
12d9561ba1 | ||
|
|
fe09879edd | ||
|
|
7104767ff9 | ||
|
|
17f8885fb1 | ||
|
|
549aae67cc | ||
|
|
fbb961b5b7 | ||
|
|
edcea87ea5 | ||
|
|
2c843e549b | ||
|
|
4ee0dab68f | ||
|
|
5d7be687d3 | ||
|
|
e2a8503496 | ||
|
|
a389306885 | ||
|
|
32c05897f4 | ||
|
|
2e319be569 | ||
|
|
852b227ea2 | ||
|
|
06b85586e6 | ||
|
|
fc32188fd2 | ||
|
|
9538fb1b7a | ||
|
|
5051809940 | ||
|
|
64dde6fd36 | ||
|
|
38921fb997 | ||
|
|
2d1a7ff374 | ||
|
|
361ca7872e | ||
|
|
81b7660232 | ||
|
|
7caa2636e4 | ||
|
|
1e894d378d | ||
|
|
6fb369ffcc | ||
|
|
a3500340cf | ||
|
|
b1832afd18 | ||
|
|
434a98d6ed | ||
|
|
cc1f196a2c | ||
|
|
e3dac34bb0 | ||
|
|
938135bb59 | ||
|
|
7d2bee59a3 | ||
|
|
6af238c2ea | ||
|
|
b71b33654a | ||
|
|
5d99a5f253 | ||
|
|
c0d869089d | ||
|
|
15f5a5694a | ||
|
|
14a44bff6e | ||
|
|
37d11b602f | ||
|
|
da3cb5e694 | ||
|
|
1e5232f47a | ||
|
|
dddaeb9a38 | ||
|
|
70a74f8629 | ||
|
|
8f1309fa18 | ||
|
|
b7a4eaff8d | ||
|
|
a20e27800c | ||
|
|
fa7a09a420 | ||
|
|
f983a491c0 | ||
|
|
f85dc12819 | ||
|
|
9798e506cf | ||
|
|
a60164edc4 | ||
|
|
6c7038cf1f | ||
|
|
f8e144a658 | ||
|
|
b4984dc38b | ||
|
|
21c9b34540 | ||
|
|
041cd526b7 | ||
|
|
9477fea3dc | ||
|
|
e4b19b5320 | ||
|
|
7692253636 | ||
|
|
e90603d4db | ||
|
|
dd76e9dcf7 | ||
|
|
fca989c098 | ||
|
|
7ebbf45921 | ||
|
|
829cadf5fb | ||
|
|
8f08073564 | ||
|
|
af08a7850b | ||
|
|
d2193ccd1b | ||
|
|
cda70164ba | ||
|
|
5e1cc4d981 | ||
|
|
4ee452b854 | ||
|
|
edbc9dede5 | ||
|
|
53ca4e2849 | ||
|
|
1801a58f6b | ||
|
|
011acd919e | ||
|
|
45d8b47366 | ||
|
|
c3820a0b44 | ||
|
|
c7b393901f | ||
|
|
f1e14c94c5 | ||
|
|
ddb171fca3 | ||
|
|
ec3bbc78dc | ||
|
|
1be1dc9d06 | ||
|
|
6109e0c6f1 | ||
|
|
5851ddd329 | ||
|
|
ad0cf5827b | ||
|
|
f6192aa6bf |
@@ -4,3 +4,6 @@
|
|||||||
#
|
#
|
||||||
# Prettier one-time repo-wide format pass (R2 S1b, 2026-07-21)
|
# Prettier one-time repo-wide format pass (R2 S1b, 2026-07-21)
|
||||||
51c36c2071728d2cd7ae580b7bfdf21f6867a347
|
51c36c2071728d2cd7ae580b7bfdf21f6867a347
|
||||||
|
|
||||||
|
# ruff format one-time backend + scripts pass (line-length 100, 2026-08-23)
|
||||||
|
b1832afd180866b922521148603db447020c57a4
|
||||||
|
|||||||
@@ -59,6 +59,8 @@ jobs:
|
|||||||
- run: pip install --no-cache-dir -r backend/requirements.lock -r backend/requirements-dev.txt
|
- run: pip install --no-cache-dir -r backend/requirements.lock -r backend/requirements-dev.txt
|
||||||
- name: ruff
|
- name: ruff
|
||||||
run: python -m ruff check --config backend/ruff.toml backend scripts
|
run: python -m ruff check --config backend/ruff.toml backend scripts
|
||||||
|
- name: ruff format --check
|
||||||
|
run: python -m ruff format --check --config backend/ruff.toml backend scripts
|
||||||
- name: pytest
|
- name: pytest
|
||||||
run: python -m pytest
|
run: python -m pytest
|
||||||
working-directory: backend
|
working-directory: backend
|
||||||
|
|||||||
@@ -12,12 +12,12 @@ provider.
|
|||||||
|
|
||||||
## The gate — never merge or ship red
|
## The gate — never merge or ship red
|
||||||
|
|
||||||
The local gate is **ruff + typecheck (3 tsc projects) + eslint + prettier --check + knip + vitest +
|
The local gate is **ruff (`check` + `format --check`) + typecheck (3 tsc projects) + eslint + prettier
|
||||||
pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
--check + knip + vitest + pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
||||||
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
||||||
|
|
||||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||||
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` and `pytest` (the DB-lane needs Postgres).
|
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` + `ruff format --check backend` and `pytest` (the DB-lane needs Postgres).
|
||||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||||
|
|
||||||
## Workflow
|
## Workflow
|
||||||
@@ -26,12 +26,33 @@ pytest**. It runs on every `publish`, which **refuses to build on failure — th
|
|||||||
keep commits logical and Conventional-Commit-styled.
|
keep commits logical and Conventional-Commit-styled.
|
||||||
- **No AI attribution in commits.** No `Co-Authored-By` trailer, no "generated by" lines. (This file
|
- **No AI attribution in commits.** No `Co-Authored-By` trailer, no "generated by" lines. (This file
|
||||||
being AI-written is the intended, visible exception; commits are not.)
|
being AI-written is the intended, visible exception; commits are not.)
|
||||||
- **Release is a deliberate, user-owned step** (`siftlode release` from the feature branch: merge to
|
- **Release: user-owned in timing, agent-run in execution.** The user decides *when* to ship; on their
|
||||||
`main` → publish → deploy → verify `/api/version`). Do not self-ship; bring work to a green,
|
go-ahead *you* run `siftlode release` from the feature branch (merge to `main` → publish → deploy →
|
||||||
committed state and stop for the user's go-ahead.
|
verify `/api/version`). Never self-ship — bring work to a green, committed state and stop for the
|
||||||
|
go-ahead first, then run it yourself.
|
||||||
- **Public repo.** Keep ops/topology/secrets out — those live in a separate private repo. Nothing here
|
- **Public repo.** Keep ops/topology/secrets out — those live in a separate private repo. Nothing here
|
||||||
should assume or leak them.
|
should assume or leak them.
|
||||||
|
|
||||||
|
## Collaboration — how to work with the human
|
||||||
|
|
||||||
|
The human is the product owner: they bring ideas and requirements. You are the technical lead — you own
|
||||||
|
how it gets built. Concretely:
|
||||||
|
|
||||||
|
- **Plan before you build.** For anything non-trivial, present a short plan (sprint-level for larger
|
||||||
|
work) and get a go-ahead before writing code.
|
||||||
|
- **Offer choices, mark the optimal — and explain the trade-off.** When several solutions are viable,
|
||||||
|
list them and mark the recommended one. This is the *one* place to spend extra words: give the *why*
|
||||||
|
behind each option and the weight of the decision. These decision points are where the product owner
|
||||||
|
engages with the build and learns from it, so make them understandable — a reasoned menu, not a bare one.
|
||||||
|
- **Mockups for design/UX decisions.** For any visual or interaction choice, show mockup options to
|
||||||
|
choose from before building.
|
||||||
|
- **Challenge unsound requests.** If a request is technically ill-advised and you can back it with real
|
||||||
|
arguments, push back or propose an alternative — don't comply blindly.
|
||||||
|
- **Verify UI in the browser before calling it done.** tsc/unit/DOM tests miss visual and interaction
|
||||||
|
bugs; do a real visual check of any UI change.
|
||||||
|
- **Be terse everywhere else.** Outside those decision points, lead with the answer or decision. During
|
||||||
|
implementation and debugging especially: no restating the same point reworded, no filler.
|
||||||
|
|
||||||
## Architecture shape (don't fight it)
|
## Architecture shape (don't fight it)
|
||||||
|
|
||||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||||
@@ -79,5 +100,6 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
*A fuller written-conventions doc (`docs/development.md`) is planned; once it lands, this file slims to a
|
*Run-it-locally and the full gate commands live in [`docs/development.md`](docs/development.md); backend
|
||||||
pointer plus the gate/workflow/no-attribution rules. Until then, the table above is the source of truth.*
|
code conventions in [`backend/app/README.md`](backend/app/README.md). This file stays the source of
|
||||||
|
truth for the canonical-patterns table and the gate/workflow/collaboration rules above.*
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0001_initial
|
|||||||
Revises:
|
Revises:
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0002_ingest_core
|
|||||||
Revises: 0001_initial
|
Revises: 0001_initial
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0003_tags
|
|||||||
Revises: 0002_ingest_core
|
Revises: 0002_ingest_core
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0004_feed_state_prefs
|
|||||||
Revises: 0003_tags
|
Revises: 0003_tags
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0005_shorts_probed
|
|||||||
Revises: 0004_feed_state_prefs
|
Revises: 0004_feed_state_prefs
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0006_app_state
|
|||||||
Revises: 0005_shorts_probed
|
Revises: 0005_shorts_probed
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0007_deep_requested
|
|||||||
Revises: 0006_app_state
|
Revises: 0006_app_state
|
||||||
Create Date: 2026-06-11
|
Create Date: 2026-06-11
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0008_invites
|
|||||||
Revises: 0007_deep_requested
|
Revises: 0007_deep_requested
|
||||||
Create Date: 2026-06-12
|
Create Date: 2026-06-12
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -22,9 +23,7 @@ def upgrade() -> None:
|
|||||||
"invites",
|
"invites",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("email", sa.String(length=320), nullable=False),
|
sa.Column("email", sa.String(length=320), nullable=False),
|
||||||
sa.Column(
|
sa.Column("status", sa.String(length=16), nullable=False, server_default="pending"),
|
||||||
"status", sa.String(length=16), nullable=False, server_default="pending"
|
|
||||||
),
|
|
||||||
sa.Column("note", sa.Text(), nullable=True),
|
sa.Column("note", sa.Text(), nullable=True),
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"requested_at",
|
"requested_at",
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0009_quota_events
|
|||||||
Revises: 0008_invites
|
Revises: 0008_invites
|
||||||
Create Date: 2026-06-12
|
Create Date: 2026-06-12
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0010_watch_progress
|
|||||||
Revises: 0009_quota_events
|
Revises: 0009_quota_events
|
||||||
Create Date: 2026-06-14
|
Create Date: 2026-06-14
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0011_playlists
|
|||||||
Revises: 0010_watch_progress
|
Revises: 0010_watch_progress
|
||||||
Create Date: 2026-06-15
|
Create Date: 2026-06-15
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -26,16 +27,10 @@ def upgrade() -> None:
|
|||||||
nullable=False,
|
nullable=False,
|
||||||
),
|
),
|
||||||
sa.Column("name", sa.String(length=255), nullable=False),
|
sa.Column("name", sa.String(length=255), nullable=False),
|
||||||
sa.Column(
|
sa.Column("kind", sa.String(length=16), nullable=False, server_default="user"),
|
||||||
"kind", sa.String(length=16), nullable=False, server_default="user"
|
sa.Column("source", sa.String(length=16), nullable=False, server_default="local"),
|
||||||
),
|
|
||||||
sa.Column(
|
|
||||||
"source", sa.String(length=16), nullable=False, server_default="local"
|
|
||||||
),
|
|
||||||
sa.Column("yt_playlist_id", sa.String(length=64), nullable=True),
|
sa.Column("yt_playlist_id", sa.String(length=64), nullable=True),
|
||||||
sa.Column(
|
sa.Column("dirty", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
"dirty", sa.Boolean(), nullable=False, server_default="false"
|
|
||||||
),
|
|
||||||
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"created_at",
|
"created_at",
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ Create Date: 2026-06-15
|
|||||||
Migrates the old per-video status='saved' into membership of a built-in 'watch_later'
|
Migrates the old per-video status='saved' into membership of a built-in 'watch_later'
|
||||||
playlist (one per user who had saved videos), then demotes those states to 'new'.
|
playlist (one per user who had saved videos), then demotes those states to 'new'.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ a YouTube read. Backfills the baseline for already-clean linked playlists (the c
|
|||||||
state == the synced state for them); dirty ones are left NULL so they stay dirty until the
|
state == the synced state for them); dirty ones are left NULL so they stay dirty until the
|
||||||
next real sync establishes a baseline.
|
next real sync establishes a baseline.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
@@ -32,24 +33,18 @@ def _fingerprint(name: str, ids: list[str]) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column(
|
op.add_column("playlists", sa.Column("synced_fingerprint", sa.String(), nullable=True))
|
||||||
"playlists", sa.Column("synced_fingerprint", sa.String(), nullable=True)
|
|
||||||
)
|
|
||||||
bind = op.get_bind()
|
bind = op.get_bind()
|
||||||
# Backfill clean, YouTube-linked playlists: their current state IS the synced state.
|
# Backfill clean, YouTube-linked playlists: their current state IS the synced state.
|
||||||
rows = bind.execute(
|
rows = bind.execute(
|
||||||
sa.text(
|
sa.text("SELECT id, name FROM playlists WHERE yt_playlist_id IS NOT NULL AND dirty = false")
|
||||||
"SELECT id, name FROM playlists "
|
|
||||||
"WHERE yt_playlist_id IS NOT NULL AND dirty = false"
|
|
||||||
)
|
|
||||||
).fetchall()
|
).fetchall()
|
||||||
for pid, name in rows:
|
for pid, name in rows:
|
||||||
ids = [
|
ids = [
|
||||||
r[0]
|
r[0]
|
||||||
for r in bind.execute(
|
for r in bind.execute(
|
||||||
sa.text(
|
sa.text(
|
||||||
"SELECT video_id FROM playlist_items "
|
"SELECT video_id FROM playlist_items WHERE playlist_id = :pid ORDER BY position"
|
||||||
"WHERE playlist_id = :pid ORDER BY position"
|
|
||||||
),
|
),
|
||||||
{"pid": pid},
|
{"pid": pid},
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ Adds the shared demo-account plumbing:
|
|||||||
The demo user row itself is created lazily on first demo login (works across all three DBs
|
The demo user row itself is created lazily on first demo login (works across all three DBs
|
||||||
without a data migration), so this only adds the schema.
|
without a data migration), so this only adds the schema.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -47,9 +48,7 @@ def upgrade() -> None:
|
|||||||
nullable=False,
|
nullable=False,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
op.create_index(
|
op.create_index("ix_demo_whitelist_email", "demo_whitelist", ["email"], unique=True)
|
||||||
"ix_demo_whitelist_email", "demo_whitelist", ["email"], unique=True
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ Create Date: 2026-06-16
|
|||||||
Adds scheduler_settings: a per-job run-interval override (minutes), editable from the admin
|
Adds scheduler_settings: a per-job run-interval override (minutes), editable from the admin
|
||||||
Scheduler dashboard. Absent row = use the env/config default, so this is purely additive.
|
Scheduler dashboard. Absent row = use the env/config default, so this is purely additive.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ the validation lifecycle (last_checked_at for rolling re-validation; unavailable
|
|||||||
unavailable_reason to flag a video as currently unplayable — hidden from the feed
|
unavailable_reason to flag a video as currently unplayable — hidden from the feed
|
||||||
immediately, hard-deleted after a grace period). Purely additive; all columns nullable.
|
immediately, hard-deleted after a grace period). Purely additive; all columns nullable.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -26,7 +27,9 @@ def upgrade() -> None:
|
|||||||
op.add_column("videos", sa.Column("privacy_status", sa.String(length=16), nullable=True))
|
op.add_column("videos", sa.Column("privacy_status", sa.String(length=16), nullable=True))
|
||||||
op.add_column("videos", sa.Column("upload_status", sa.String(length=16), nullable=True))
|
op.add_column("videos", sa.Column("upload_status", sa.String(length=16), nullable=True))
|
||||||
op.add_column("videos", sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True))
|
op.add_column("videos", sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True))
|
||||||
op.add_column("videos", sa.Column("unavailable_since", sa.DateTime(timezone=True), nullable=True))
|
op.add_column(
|
||||||
|
"videos", sa.Column("unavailable_since", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
op.add_column("videos", sa.Column("unavailable_reason", sa.String(length=24), nullable=True))
|
op.add_column("videos", sa.Column("unavailable_reason", sa.String(length=24), nullable=True))
|
||||||
op.create_index("ix_videos_last_checked_at", "videos", ["last_checked_at"])
|
op.create_index("ix_videos_last_checked_at", "videos", ["last_checked_at"])
|
||||||
op.create_index("ix_videos_unavailable_since", "videos", ["unavailable_since"])
|
op.create_index("ix_videos_unavailable_since", "videos", ["unavailable_since"])
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ JSON payload column is named `data` (SQLAlchemy's declarative Base reserves `met
|
|||||||
First producer is the maintenance job's batched "videos removed" notice. Coexists with the
|
First producer is the maintenance job's batched "videos removed" notice. Coexists with the
|
||||||
existing client-side transient bell.
|
existing client-side transient bell.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -36,18 +37,14 @@ def upgrade() -> None:
|
|||||||
sa.Column("data", sa.JSON(), nullable=True),
|
sa.Column("data", sa.JSON(), nullable=True),
|
||||||
sa.Column("read", sa.Boolean(), nullable=False, server_default="false"),
|
sa.Column("read", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
sa.Column("dismissed", sa.Boolean(), nullable=False, server_default="false"),
|
sa.Column("dismissed", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
sa.Column(
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now()
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
op.create_index("ix_notifications_user_id", "notifications", ["user_id"])
|
op.create_index("ix_notifications_user_id", "notifications", ["user_id"])
|
||||||
op.create_index("ix_notifications_type", "notifications", ["type"])
|
op.create_index("ix_notifications_type", "notifications", ["type"])
|
||||||
op.create_index("ix_notifications_read", "notifications", ["read"])
|
op.create_index("ix_notifications_read", "notifications", ["read"])
|
||||||
op.create_index("ix_notifications_created_at", "notifications", ["created_at"])
|
op.create_index("ix_notifications_created_at", "notifications", ["created_at"])
|
||||||
# Fast unread-badge count + inbox listing per user.
|
# Fast unread-badge count + inbox listing per user.
|
||||||
op.create_index(
|
op.create_index("ix_notifications_user_read", "notifications", ["user_id", "read"])
|
||||||
"ix_notifications_user_read", "notifications", ["user_id", "read"]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds app_state.maintenance_revalidate_batch: an admin-tunable override (from the
|
|||||||
dashboard) for how many videos the maintenance job re-validates per run. NULL = use the
|
dashboard) for how many videos the maintenance job re-validates per run. NULL = use the
|
||||||
env/config default. Purely additive.
|
env/config default. Purely additive.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ channels.list?mine=true and cached. Used to exclude the user's own channel from
|
|||||||
playlist-discovery list (you don't subscribe to yourself). NULL = not resolved yet.
|
playlist-discovery list (you don't subscribe to yourself). NULL = not resolved yet.
|
||||||
Purely additive.
|
Purely additive.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ stored historical values to the canonical scheme defined in app.quota.QuotaActio
|
|||||||
fully reversible. (Scheduler job ids and progress phase labels are a separate namespace and
|
fully reversible. (Scheduler job ids and progress phase labels are a separate namespace and
|
||||||
are NOT touched.)
|
are NOT touched.)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds the system_config table: admin overrides for operational config that otherw
|
|||||||
from env/config defaults (see app.sysconfig for the registry of known keys). Secret values
|
from env/config defaults (see app.sysconfig for the registry of known keys). Secret values
|
||||||
are stored Fernet-encrypted (encrypted flag). Purely additive; absent rows = use defaults.
|
are stored Fernet-encrypted (encrypted flag). Purely additive; absent rows = use defaults.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ google_sub nullable (password-only accounts have no Google sub until they link),
|
|||||||
single-use auth_tokens table for email verification + password reset. Existing rows are all
|
single-use auth_tokens table for email verification + password reset. Existing rows are all
|
||||||
Google/demo accounts → backfilled email_verified=true (is_active defaults true).
|
Google/demo accounts → backfilled email_verified=true (is_active defaults true).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds users.is_suspended — an admin-controlled access block, distinct from is_a
|
|||||||
approval lifecycle). A suspended account can't sign in (password or Google) and any live
|
approval lifecycle). A suspended account can't sign in (password or Google) and any live
|
||||||
session is rejected; the user is told why and given the operator's contact.
|
session is rejected; the user is told why and given the operator's contact.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ email_verified=false (the callback didn't set it). Backfill those so the flag re
|
|||||||
matters because password-login gates on email_verified, so a Google user who later adds a password
|
matters because password-login gates on email_verified, so a Google user who later adds a password
|
||||||
must count as verified. Demo/password-only rows (google_sub IS NULL) are left untouched.
|
must count as verified. Demo/password-only rows (google_sub IS NULL) are left untouched.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds app_state.configured + setup_token_hash for the first-run install wizard. E
|
|||||||
(any users present) are marked configured so they never see the wizard; a genuinely fresh DB (no
|
(any users present) are marked configured so they never see the wizard; a genuinely fresh DB (no
|
||||||
users) starts unconfigured and boots into setup mode.
|
users) starts unconfigured and boots into setup mode.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds the `messages` table for direct user-to-user messaging (notification module
|
|||||||
A conversation is the set of rows between two users; both FKs cascade so a GDPR account
|
A conversation is the set of rows between two users; both FKs cascade so a GDPR account
|
||||||
erasure removes that user's messages.
|
erasure removes that user's messages.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ hold each user's public key + passphrase-wrapped private key (the server can nev
|
|||||||
Any pre-existing rows were plaintext with no keys, so they're dropped (the feature is unshipped;
|
Any pre-existing rows were plaintext with no keys, so they're dropped (the feature is unshipped;
|
||||||
prod has none).
|
prod has none).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ the Library by default:
|
|||||||
- channels.from_search — this channel exists only because search surfaced one of its videos
|
- channels.from_search — this channel exists only because search surfaced one of its videos
|
||||||
Both default false (every existing row is organic), indexed for the feed's exclude filter.
|
Both default false (every existing row is organic), indexed for the feed's exclude filter.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -25,16 +26,12 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column(
|
op.add_column(
|
||||||
"videos",
|
"videos",
|
||||||
sa.Column(
|
sa.Column("via_search", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
"via_search", sa.Boolean(), nullable=False, server_default="false"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
op.create_index("ix_videos_via_search", "videos", ["via_search"])
|
op.create_index("ix_videos_via_search", "videos", ["via_search"])
|
||||||
op.add_column(
|
op.add_column(
|
||||||
"channels",
|
"channels",
|
||||||
sa.Column(
|
sa.Column("from_search", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
"from_search", sa.Boolean(), nullable=False, server_default="false"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
op.create_index("ix_channels_from_search", "channels", ["from_search"])
|
op.create_index("ix_channels_from_search", "channels", ["from_search"])
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ query wraps both sides in unaccent(); this migration just makes the function ava
|
|||||||
`unaccent` ships with the official postgres image (postgresql-contrib). The DB role is a
|
`unaccent` ships with the official postgres image (postgresql-contrib). The DB role is a
|
||||||
superuser in our deployments, so CREATE EXTENSION succeeds; IF NOT EXISTS keeps it idempotent.
|
superuser in our deployments, so CREATE EXTENSION succeeds; IF NOT EXISTS keeps it idempotent.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ live YouTube search. This lets "videos I searched for" appear in that user's Min
|
|||||||
Mine Source filter), independent of the global `videos.via_search` flag (which marks a video as
|
Mine Source filter), independent of the global `videos.via_search` flag (which marks a video as
|
||||||
search-discovered by anyone, for the shared Library's Source filter).
|
search-discovered by anyone, for the shared Library's Source filter).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ search stays accent-insensitive ("tiesto" matches "Tiësto"). A GIN expression i
|
|||||||
query uses the exact same expression so the planner picks the index. `unaccent` is already
|
query uses the exact same expression so the planner picks the index. `unaccent` is already
|
||||||
enabled (migration 0029). The config-based 2-arg to_tsvector is IMMUTABLE, so it's index-safe.
|
enabled (migration 0029). The config-based 2-arg to_tsvector is IMMUTABLE, so it's index-safe.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -26,9 +27,7 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.execute(
|
op.execute("CREATE TEXT SEARCH CONFIGURATION public.unaccent_simple (COPY = pg_catalog.simple)")
|
||||||
"CREATE TEXT SEARCH CONFIGURATION public.unaccent_simple (COPY = pg_catalog.simple)"
|
|
||||||
)
|
|
||||||
op.execute(
|
op.execute(
|
||||||
"""
|
"""
|
||||||
ALTER TEXT SEARCH CONFIGURATION public.unaccent_simple
|
ALTER TEXT SEARCH CONFIGURATION public.unaccent_simple
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ title-only expression index from 0031 is dropped (superseded). Existing rows get
|
|||||||
description indexed immediately; keywords/search_terms fill in as videos are (re-)enriched and
|
description indexed immediately; keywords/search_terms fill in as videos are (re-)enriched and
|
||||||
searched. Uses the `unaccent_simple` config from 0031 (accent-insensitive).
|
searched. Uses the `unaccent_simple` config from 0031 (accent-insensitive).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -48,9 +49,7 @@ def upgrade() -> None:
|
|||||||
f"ALTER TABLE videos ADD COLUMN search_vector tsvector "
|
f"ALTER TABLE videos ADD COLUMN search_vector tsvector "
|
||||||
f"GENERATED ALWAYS AS ({_SEARCH_VECTOR_EXPR}) STORED"
|
f"GENERATED ALWAYS AS ({_SEARCH_VECTOR_EXPR}) STORED"
|
||||||
)
|
)
|
||||||
op.execute(
|
op.execute("CREATE INDEX ix_videos_search_vector ON videos USING gin (search_vector)")
|
||||||
"CREATE INDEX ix_videos_search_vector ON videos USING gin (search_vector)"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ un-subscribed channels):
|
|||||||
access to its ephemeral videos and drives the grace-period purge of un-kept channels.
|
access to its ephemeral videos and drives the grace-period purge of un-kept channels.
|
||||||
All new flags default false / NULL, so every existing row stays organic.
|
All new flags default false / NULL, so every existing row stays organic.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -30,9 +31,7 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column("channels", sa.Column("total_view_count", sa.BigInteger(), nullable=True))
|
op.add_column("channels", sa.Column("total_view_count", sa.BigInteger(), nullable=True))
|
||||||
op.add_column(
|
op.add_column("channels", sa.Column("published_at", sa.DateTime(timezone=True), nullable=True))
|
||||||
"channels", sa.Column("published_at", sa.DateTime(timezone=True), nullable=True)
|
|
||||||
)
|
|
||||||
op.add_column("channels", sa.Column("banner_url", sa.String(length=1024), nullable=True))
|
op.add_column("channels", sa.Column("banner_url", sa.String(length=1024), nullable=True))
|
||||||
op.add_column("channels", sa.Column("external_links", sa.JSON(), nullable=True))
|
op.add_column("channels", sa.Column("external_links", sa.JSON(), nullable=True))
|
||||||
op.add_column(
|
op.add_column(
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds the `blocked_channels` table: a per-user blocklist. Videos from a blocked c
|
|||||||
never shown in that user's live YouTube search (and not ingested), and are hidden from their
|
never shown in that user's live YouTube search (and not ingested), and are hidden from their
|
||||||
feed / explore views. Per-user, so one user's block doesn't affect anyone else.
|
feed / explore views. Per-user, so one user's block doesn't affect anyone else.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Adds the `saved_views` table: per-user named snapshots of the feed's filter/sort
|
|||||||
row stores the serialized FeedFilters (JSON), a sidebar `position` for drag-reorder, and an
|
row stores the serialized FeedFilters (JSON), a sidebar `position` for drag-reorder, and an
|
||||||
`is_default` flag (at most one per user, enforced by the API) for the view applied on load.
|
`is_default` flag (at most one per user, enforced by the API) for the view applied on load.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -26,9 +27,7 @@ def upgrade() -> None:
|
|||||||
sa.Column("user_id", sa.Integer(), nullable=False),
|
sa.Column("user_id", sa.Integer(), nullable=False),
|
||||||
sa.Column("name", sa.String(length=80), nullable=False),
|
sa.Column("name", sa.String(length=80), nullable=False),
|
||||||
sa.Column("filters", sa.JSON(), nullable=False),
|
sa.Column("filters", sa.JSON(), nullable=False),
|
||||||
sa.Column(
|
sa.Column("position", sa.Integer(), server_default=sa.text("0"), nullable=False),
|
||||||
"position", sa.Integer(), server_default=sa.text("0"), nullable=False
|
|
||||||
),
|
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"is_default",
|
"is_default",
|
||||||
sa.Boolean(),
|
sa.Boolean(),
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ FOR UPDATE SKIP LOCKED, live progress written back for polling); `download_profi
|
|||||||
reusable format presets (builtins seeded in the next migration); `download_shares` grant a file
|
reusable format presets (builtins seeded in the next migration); `download_shares` grant a file
|
||||||
to another user; `download_quotas` hold admin-set per-user limits (default via sysconfig).
|
to another user; `download_quotas` hold admin-set per-user limits (default via sysconfig).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -47,9 +48,16 @@ def upgrade() -> None:
|
|||||||
sa.Column("ref_count", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("ref_count", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("last_access_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("last_access_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
sa.UniqueConstraint("source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"),
|
sa.UniqueConstraint(
|
||||||
|
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_media_assets_status", "media_assets", ["status"])
|
op.create_index("ix_media_assets_status", "media_assets", ["status"])
|
||||||
|
|
||||||
@@ -61,7 +69,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("spec", sa.JSON(), nullable=False),
|
sa.Column("spec", sa.JSON(), nullable=False),
|
||||||
sa.Column("is_builtin", sa.Boolean(), server_default="false", nullable=False),
|
sa.Column("is_builtin", sa.Boolean(), server_default="false", nullable=False),
|
||||||
sa.Column("sort_order", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("sort_order", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
)
|
)
|
||||||
@@ -84,8 +97,18 @@ def upgrade() -> None:
|
|||||||
sa.Column("phase", sa.String(length=32), nullable=True),
|
sa.Column("phase", sa.String(length=32), nullable=True),
|
||||||
sa.Column("error", sa.Text(), nullable=True),
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
sa.Column("queue_pos", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("queue_pos", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["asset_id"], ["media_assets.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["asset_id"], ["media_assets.id"], ondelete="SET NULL"),
|
||||||
sa.ForeignKeyConstraint(["profile_id"], ["download_profiles.id"], ondelete="SET NULL"),
|
sa.ForeignKeyConstraint(["profile_id"], ["download_profiles.id"], ondelete="SET NULL"),
|
||||||
@@ -102,7 +125,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("job_id", sa.Integer(), nullable=False),
|
sa.Column("job_id", sa.Integer(), nullable=False),
|
||||||
sa.Column("from_user_id", sa.Integer(), nullable=False),
|
sa.Column("from_user_id", sa.Integer(), nullable=False),
|
||||||
sa.Column("to_user_id", sa.Integer(), nullable=False),
|
sa.Column("to_user_id", sa.Integer(), nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["from_user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["from_user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["to_user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["to_user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
@@ -120,7 +148,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("max_concurrent", sa.Integer(), nullable=False),
|
sa.Column("max_concurrent", sa.Integer(), nullable=False),
|
||||||
sa.Column("max_jobs", sa.Integer(), nullable=False),
|
sa.Column("max_jobs", sa.Integer(), nullable=False),
|
||||||
sa.Column("unlimited", sa.Boolean(), server_default="false", nullable=False),
|
sa.Column("unlimited", sa.Boolean(), server_default="false", nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("user_id"),
|
sa.PrimaryKeyConstraint("user_id"),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ definition consumed by app.downloads.formats: mode ("av" audio+video / "v" video
|
|||||||
the embed_*/sponsorblock postprocessor flags. Names are English (the UI default); the frontend
|
the embed_*/sponsorblock postprocessor flags. Names are English (the UI default); the frontend
|
||||||
localizes builtin labels by a stable key derived from the name.
|
localizes builtin labels by a stable key derived from the name.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ Create Date: 2026-07-03
|
|||||||
The download GC warns each holder once, a grace window before an asset's TTL expiry; this flag
|
The download GC warns each holder once, a grace window before an asset's TTL expiry; this flag
|
||||||
records that so subsequent GC runs don't re-notify until the asset is deleted.
|
records that so subsequent GC runs don't re-notify until the asset is deleted.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ display-friendly form (emoji stripped, ALL-CAPS de-shouted, trailing SEO hashtag
|
|||||||
see app.titles). Reversible: original_title preserves the source, and title can be re-derived.
|
see app.titles). Reversible: original_title preserves the source, and title can be re-derived.
|
||||||
The generated search_vector column regenerates automatically as each title is updated.
|
The generated search_vector column regenerates automatically as each title is updated.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -45,5 +46,7 @@ def upgrade() -> None:
|
|||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
conn = op.get_bind()
|
conn = op.get_bind()
|
||||||
conn.execute(sa.text("UPDATE videos SET title = original_title WHERE original_title IS NOT NULL"))
|
conn.execute(
|
||||||
|
sa.text("UPDATE videos SET title = original_title WHERE original_title IS NOT NULL")
|
||||||
|
)
|
||||||
op.drop_column("videos", "original_title")
|
op.drop_column("videos", "original_title")
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ rows so both fresh installs (0037 then this) and existing DBs converge on the sa
|
|||||||
jobs are unaffected (they snapshot their spec; profile_id just SET NULLs if it pointed at a
|
jobs are unaffected (they snapshot their spec; profile_id just SET NULLs if it pointed at a
|
||||||
replaced row).
|
replaced row).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ still hangs off a `media_assets` row (with `source_kind='edit'` and a per-user c
|
|||||||
existing file-serve / ref-count / GC / quota machinery is reused unchanged. `source_job_id` /
|
existing file-serve / ref-count / GC / quota machinery is reused unchanged. `source_job_id` /
|
||||||
`source_asset_id` point at the parent download; `edit_spec` is the trim/crop recipe.
|
`source_asset_id` point at the parent download; `edit_spec` is the trim/crop recipe.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -24,16 +25,10 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column(
|
op.add_column(
|
||||||
"download_jobs",
|
"download_jobs",
|
||||||
sa.Column(
|
sa.Column("job_kind", sa.String(length=16), server_default="download", nullable=False),
|
||||||
"job_kind", sa.String(length=16), server_default="download", nullable=False
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"download_jobs", sa.Column("source_job_id", sa.Integer(), nullable=True)
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"download_jobs", sa.Column("source_asset_id", sa.Integer(), nullable=True)
|
|
||||||
)
|
)
|
||||||
|
op.add_column("download_jobs", sa.Column("source_job_id", sa.Integer(), nullable=True))
|
||||||
|
op.add_column("download_jobs", sa.Column("source_asset_id", sa.Integer(), nullable=True))
|
||||||
op.add_column("download_jobs", sa.Column("edit_spec", sa.JSON(), nullable=True))
|
op.add_column("download_jobs", sa.Column("edit_spec", sa.JSON(), nullable=True))
|
||||||
op.create_foreign_key(
|
op.create_foreign_key(
|
||||||
"fk_download_jobs_source_job",
|
"fk_download_jobs_source_job",
|
||||||
@@ -54,12 +49,8 @@ def upgrade() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.drop_constraint(
|
op.drop_constraint("fk_download_jobs_source_asset", "download_jobs", type_="foreignkey")
|
||||||
"fk_download_jobs_source_asset", "download_jobs", type_="foreignkey"
|
op.drop_constraint("fk_download_jobs_source_job", "download_jobs", type_="foreignkey")
|
||||||
)
|
|
||||||
op.drop_constraint(
|
|
||||||
"fk_download_jobs_source_job", "download_jobs", type_="foreignkey"
|
|
||||||
)
|
|
||||||
op.drop_column("download_jobs", "edit_spec")
|
op.drop_column("download_jobs", "edit_spec")
|
||||||
op.drop_column("download_jobs", "source_asset_id")
|
op.drop_column("download_jobs", "source_asset_id")
|
||||||
op.drop_column("download_jobs", "source_job_id")
|
op.drop_column("download_jobs", "source_job_id")
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ can watch the file on the public `/watch/{token}` player page — no account nee
|
|||||||
per-link controls: an expiry, an "allow download" toggle, and an (argon2-hashed) password. Revoke
|
per-link controls: an expiry, an "allow download" toggle, and an (argon2-hashed) password. Revoke
|
||||||
= delete the row. Distinct from `download_shares` (an ACL grant to another *registered* user).
|
= delete the row. Distinct from `download_shares` (an ACL grant to another *registered* user).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -31,7 +32,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("password_hash", sa.Text(), nullable=True),
|
sa.Column("password_hash", sa.Text(), nullable=True),
|
||||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("view_count", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("view_count", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["job_id"], ["download_jobs.id"], ondelete="CASCADE"),
|
||||||
sa.ForeignKeyConstraint(["created_by"], ["users.id"], ondelete="CASCADE"),
|
sa.ForeignKeyConstraint(["created_by"], ["users.id"], ondelete="CASCADE"),
|
||||||
sa.PrimaryKeyConstraint("id"),
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Stores yt-dlp's canonical `webpage_url` for a download's source, so the Download
|
|||||||
clean "downloaded from" reference (copyable + openable). Nullable — filled by the worker on
|
clean "downloaded from" reference (copyable + openable). Nullable — filled by the worker on
|
||||||
extraction; existing/queued rows fall back to a URL derived from source_kind+source_ref.
|
extraction; existing/queued rows fall back to a URL derived from source_kind+source_ref.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ Plex server (playback is from the local physical file; Plex is metadata only). T
|
|||||||
The generated `search_vector` columns + their GIN indexes are added via raw SQL (same approach
|
The generated `search_vector` columns + their GIN indexes are added via raw SQL (same approach
|
||||||
as migration 0032), reusing the `unaccent_simple` text-search config from migration 0031.
|
as migration 0032), reusing the `unaccent_simple` text-search config from migration 0031.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -52,8 +53,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("kind", sa.String(length=16), nullable=False),
|
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||||
sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False),
|
sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False),
|
||||||
sa.Column("synced_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("synced_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_libraries_plex_key", "plex_libraries", ["plex_key"], unique=True)
|
op.create_index("ix_plex_libraries_plex_key", "plex_libraries", ["plex_key"], unique=True)
|
||||||
|
|
||||||
@@ -61,7 +66,12 @@ def upgrade() -> None:
|
|||||||
"plex_shows",
|
"plex_shows",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
||||||
sa.Column("library_id", sa.Integer(), sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"library_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("title", sa.String(length=512), nullable=False),
|
sa.Column("title", sa.String(length=512), nullable=False),
|
||||||
sa.Column("summary", sa.Text(), nullable=True),
|
sa.Column("summary", sa.Text(), nullable=True),
|
||||||
sa.Column("year", sa.Integer(), nullable=True),
|
sa.Column("year", sa.Integer(), nullable=True),
|
||||||
@@ -69,8 +79,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("art_key", sa.String(length=512), nullable=True),
|
sa.Column("art_key", sa.String(length=512), nullable=True),
|
||||||
sa.Column("child_count", sa.Integer(), nullable=True),
|
sa.Column("child_count", sa.Integer(), nullable=True),
|
||||||
sa.Column("added_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("added_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_shows_rating_key", "plex_shows", ["rating_key"], unique=True)
|
op.create_index("ix_plex_shows_rating_key", "plex_shows", ["rating_key"], unique=True)
|
||||||
op.create_index("ix_plex_shows_library_id", "plex_shows", ["library_id"])
|
op.create_index("ix_plex_shows_library_id", "plex_shows", ["library_id"])
|
||||||
@@ -81,12 +95,21 @@ def upgrade() -> None:
|
|||||||
"plex_seasons",
|
"plex_seasons",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
||||||
sa.Column("show_id", sa.Integer(), sa.ForeignKey("plex_shows.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"show_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_shows.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("title", sa.String(length=255), nullable=True),
|
sa.Column("title", sa.String(length=255), nullable=True),
|
||||||
sa.Column("season_number", sa.Integer(), nullable=True),
|
sa.Column("season_number", sa.Integer(), nullable=True),
|
||||||
sa.Column("thumb_key", sa.String(length=512), nullable=True),
|
sa.Column("thumb_key", sa.String(length=512), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_seasons_rating_key", "plex_seasons", ["rating_key"], unique=True)
|
op.create_index("ix_plex_seasons_rating_key", "plex_seasons", ["rating_key"], unique=True)
|
||||||
op.create_index("ix_plex_seasons_show_id", "plex_seasons", ["show_id"])
|
op.create_index("ix_plex_seasons_show_id", "plex_seasons", ["show_id"])
|
||||||
@@ -96,10 +119,25 @@ def upgrade() -> None:
|
|||||||
"plex_items",
|
"plex_items",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
||||||
sa.Column("library_id", sa.Integer(), sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"library_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("kind", sa.String(length=16), nullable=False),
|
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||||
sa.Column("show_id", sa.Integer(), sa.ForeignKey("plex_shows.id", ondelete="CASCADE"), nullable=True),
|
sa.Column(
|
||||||
sa.Column("season_id", sa.Integer(), sa.ForeignKey("plex_seasons.id", ondelete="CASCADE"), nullable=True),
|
"show_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_shows.id", ondelete="CASCADE"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"season_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_seasons.id", ondelete="CASCADE"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
sa.Column("season_number", sa.Integer(), nullable=True),
|
sa.Column("season_number", sa.Integer(), nullable=True),
|
||||||
sa.Column("episode_number", sa.Integer(), nullable=True),
|
sa.Column("episode_number", sa.Integer(), nullable=True),
|
||||||
sa.Column("title", sa.String(length=512), nullable=False),
|
sa.Column("title", sa.String(length=512), nullable=False),
|
||||||
@@ -119,8 +157,12 @@ def upgrade() -> None:
|
|||||||
sa.Column("probed_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("probed_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("markers", sa.JSON(), nullable=True),
|
sa.Column("markers", sa.JSON(), nullable=True),
|
||||||
sa.Column("added_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("added_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_items_rating_key", "plex_items", ["rating_key"], unique=True)
|
op.create_index("ix_plex_items_rating_key", "plex_items", ["rating_key"], unique=True)
|
||||||
op.create_index("ix_plex_items_library_id", "plex_items", ["library_id"])
|
op.create_index("ix_plex_items_library_id", "plex_items", ["library_id"])
|
||||||
@@ -137,14 +179,23 @@ def upgrade() -> None:
|
|||||||
op.create_table(
|
op.create_table(
|
||||||
"plex_states",
|
"plex_states",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
sa.Column("item_id", sa.Integer(), sa.ForeignKey("plex_items.id", ondelete="CASCADE"), nullable=False),
|
"user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"item_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_items.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("status", sa.String(length=12), server_default="new", nullable=False),
|
sa.Column("status", sa.String(length=12), server_default="new", nullable=False),
|
||||||
sa.Column("position_seconds", sa.Integer(), server_default="0", nullable=False),
|
sa.Column("position_seconds", sa.Integer(), server_default="0", nullable=False),
|
||||||
sa.Column("watched_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("watched_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("progress_updated_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("progress_updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("synced_to_plex", sa.Boolean(), server_default="false", nullable=False),
|
sa.Column("synced_to_plex", sa.Boolean(), server_default="false", nullable=False),
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
sa.UniqueConstraint("user_id", "item_id", name="uq_plex_user_item"),
|
sa.UniqueConstraint("user_id", "item_id", name="uq_plex_user_item"),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_states_user_id", "plex_states", ["user_id"])
|
op.create_index("ix_plex_states_user_id", "plex_states", ["user_id"])
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ Plex section listing (no per-item API calls): rating (audienceRating ~ IMDb), co
|
|||||||
release date, and GIN-indexed genre / director / cast arrays. These also double as the content
|
release date, and GIN-indexed genre / director / cast arrays. These also double as the content
|
||||||
signals a future watch-habit recommender would use.
|
signals a future watch-habit recommender would use.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -33,9 +34,13 @@ def upgrade() -> None:
|
|||||||
op.create_index("ix_plex_items_rating", "plex_items", ["rating"])
|
op.create_index("ix_plex_items_rating", "plex_items", ["rating"])
|
||||||
op.create_index("ix_plex_items_content_rating", "plex_items", ["content_rating"])
|
op.create_index("ix_plex_items_content_rating", "plex_items", ["content_rating"])
|
||||||
op.create_index("ix_plex_items_studio", "plex_items", ["studio"])
|
op.create_index("ix_plex_items_studio", "plex_items", ["studio"])
|
||||||
op.create_index("ix_plex_items_originally_available_at", "plex_items", ["originally_available_at"])
|
op.create_index(
|
||||||
|
"ix_plex_items_originally_available_at", "plex_items", ["originally_available_at"]
|
||||||
|
)
|
||||||
op.create_index("ix_plex_items_genres_gin", "plex_items", ["genres"], postgresql_using="gin")
|
op.create_index("ix_plex_items_genres_gin", "plex_items", ["genres"], postgresql_using="gin")
|
||||||
op.create_index("ix_plex_items_directors_gin", "plex_items", ["directors"], postgresql_using="gin")
|
op.create_index(
|
||||||
|
"ix_plex_items_directors_gin", "plex_items", ["directors"], postgresql_using="gin"
|
||||||
|
)
|
||||||
op.create_index("ix_plex_items_cast_gin", "plex_items", ["cast_names"], postgresql_using="gin")
|
op.create_index("ix_plex_items_cast_gin", "plex_items", ["cast_names"], postgresql_using="gin")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ finds titles by an actor/director name and ranks those above films that merely m
|
|||||||
their synopsis. A generated column's expression can't be altered in place, so we drop + recreate it
|
their synopsis. A generated column's expression can't be altered in place, so we drop + recreate it
|
||||||
(and its GIN index). `people_text` is populated on the next Plex sync.
|
(and its GIN index). `people_text` is populated on the next Plex sync.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ A `plex_collections` table holds each collection's card metadata; membership is
|
|||||||
combinable filter and a title's "collection strip" is a pure local lookup. `editable` marks the
|
combinable filter and a title's "collection strip" is a pure local lookup. `editable` marks the
|
||||||
(curated) collections we may later write back to Plex; everything is read-only by default.
|
(curated) collections we may later write back to Plex; everything is read-only by default.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -27,7 +28,12 @@ def upgrade() -> None:
|
|||||||
"plex_collections",
|
"plex_collections",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
sa.Column("rating_key", sa.String(length=32), nullable=False),
|
||||||
sa.Column("library_id", sa.Integer(), sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"library_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_libraries.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
sa.Column("title", sa.String(length=512), nullable=False),
|
sa.Column("title", sa.String(length=512), nullable=False),
|
||||||
sa.Column("summary", sa.Text(), nullable=True),
|
sa.Column("summary", sa.Text(), nullable=True),
|
||||||
sa.Column("thumb_key", sa.String(length=512), nullable=True),
|
sa.Column("thumb_key", sa.String(length=512), nullable=True),
|
||||||
@@ -35,16 +41,32 @@ def upgrade() -> None:
|
|||||||
sa.Column("smart", sa.Boolean(), nullable=False, server_default="false"),
|
sa.Column("smart", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
sa.Column("editable", sa.Boolean(), nullable=False, server_default="false"),
|
sa.Column("editable", sa.Boolean(), nullable=False, server_default="false"),
|
||||||
sa.Column("synced_at", sa.DateTime(timezone=True), nullable=True),
|
sa.Column("synced_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_plex_collections_rating_key", "plex_collections", ["rating_key"], unique=True
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_collections_rating_key", "plex_collections", ["rating_key"], unique=True)
|
|
||||||
op.create_index("ix_plex_collections_library_id", "plex_collections", ["library_id"])
|
op.create_index("ix_plex_collections_library_id", "plex_collections", ["library_id"])
|
||||||
|
|
||||||
op.add_column("plex_items", sa.Column("collection_keys", postgresql.JSONB(), nullable=True))
|
op.add_column("plex_items", sa.Column("collection_keys", postgresql.JSONB(), nullable=True))
|
||||||
op.create_index("ix_plex_items_collections_gin", "plex_items", ["collection_keys"], postgresql_using="gin")
|
op.create_index(
|
||||||
|
"ix_plex_items_collections_gin", "plex_items", ["collection_keys"], postgresql_using="gin"
|
||||||
|
)
|
||||||
op.add_column("plex_shows", sa.Column("collection_keys", postgresql.JSONB(), nullable=True))
|
op.add_column("plex_shows", sa.Column("collection_keys", postgresql.JSONB(), nullable=True))
|
||||||
op.create_index("ix_plex_shows_collections_gin", "plex_shows", ["collection_keys"], postgresql_using="gin")
|
op.create_index(
|
||||||
|
"ix_plex_shows_collections_gin", "plex_shows", ["collection_keys"], postgresql_using="gin"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ Plex account (like plex_states) — the "your own lists" counterpart to the shar
|
|||||||
collections. `plex_playlists` is owned by a user; `plex_playlist_items` holds the ordered entries
|
collections. `plex_playlists` is owned by a user; `plex_playlist_items` holds the ordered entries
|
||||||
(`position`). `plex_rating_key` is reserved for an optional later Plex-direction sync.
|
(`position`). `plex_rating_key` is reserved for an optional later Plex-direction sync.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -24,11 +25,23 @@ def upgrade() -> None:
|
|||||||
op.create_table(
|
op.create_table(
|
||||||
"plex_playlists",
|
"plex_playlists",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column("user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False),
|
sa.Column(
|
||||||
|
"user_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||||
|
),
|
||||||
sa.Column("title", sa.String(length=512), nullable=False),
|
sa.Column("title", sa.String(length=512), nullable=False),
|
||||||
sa.Column("plex_rating_key", sa.String(length=32), nullable=True),
|
sa.Column("plex_rating_key", sa.String(length=32), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_playlists_user_id", "plex_playlists", ["user_id"])
|
op.create_index("ix_plex_playlists_user_id", "plex_playlists", ["user_id"])
|
||||||
|
|
||||||
@@ -36,12 +49,16 @@ def upgrade() -> None:
|
|||||||
"plex_playlist_items",
|
"plex_playlist_items",
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"playlist_id", sa.Integer(),
|
"playlist_id",
|
||||||
sa.ForeignKey("plex_playlists.id", ondelete="CASCADE"), nullable=False,
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_playlists.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
),
|
),
|
||||||
sa.Column(
|
sa.Column(
|
||||||
"item_id", sa.Integer(),
|
"item_id",
|
||||||
sa.ForeignKey("plex_items.id", ondelete="CASCADE"), nullable=False,
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("plex_items.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
),
|
),
|
||||||
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
||||||
sa.UniqueConstraint("playlist_id", "item_id", name="uq_plex_playlist_item"),
|
sa.UniqueConstraint("playlist_id", "item_id", name="uq_plex_playlist_item"),
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ and captures yt-dlp's canonical channel/uploader URL so an auto-filled channel b
|
|||||||
overrides (like display_name — display metadata only, never touches the on-disk file).
|
overrides (like display_name — display metadata only, never touches the on-disk file).
|
||||||
- download_jobs.extra_links: optional list of extra reference URLs shown on the card + watch page.
|
- download_jobs.extra_links: optional list of extra reference URLs shown on the card + watch page.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -26,7 +27,9 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
op.add_column("media_assets", sa.Column("uploader_url", sa.Text(), nullable=True))
|
op.add_column("media_assets", sa.Column("uploader_url", sa.Text(), nullable=True))
|
||||||
op.add_column("download_jobs", sa.Column("display_uploader", sa.String(length=255), nullable=True))
|
op.add_column(
|
||||||
|
"download_jobs", sa.Column("display_uploader", sa.String(length=255), nullable=True)
|
||||||
|
)
|
||||||
op.add_column("download_jobs", sa.Column("display_uploader_url", sa.Text(), nullable=True))
|
op.add_column("download_jobs", sa.Column("display_uploader_url", sa.Text(), nullable=True))
|
||||||
op.add_column("download_jobs", sa.Column("extra_links", sa.JSON(), nullable=True))
|
op.add_column("download_jobs", sa.Column("extra_links", sa.JSON(), nullable=True))
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ download shows a blank image box. The worker now extracts a representative frame
|
|||||||
saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
|
saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
|
||||||
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
|
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ state with a Plex account. The owner MVP uses the server admin token (uses_admin
|
|||||||
token); a personal token column is reserved for later friend-account linking (PIN OAuth). See the
|
token); a personal token column is reserved for later friend-account linking (PIN OAuth). See the
|
||||||
PlexLink model docstring.
|
PlexLink model docstring.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ blob folded into the generated `search_vector` (weight B). This lets the TV grid
|
|||||||
sorted like the movie grid. A generated column's expression can't be altered in place, so the
|
sorted like the movie grid. A generated column's expression can't be altered in place, so the
|
||||||
search_vector is dropped + recreated. All new columns are populated on the next Plex sync.
|
search_vector is dropped + recreated. All new columns are populated on the next Plex sync.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -57,9 +58,13 @@ def upgrade() -> None:
|
|||||||
op.create_index("ix_plex_shows_rating", "plex_shows", ["rating"])
|
op.create_index("ix_plex_shows_rating", "plex_shows", ["rating"])
|
||||||
op.create_index("ix_plex_shows_content_rating", "plex_shows", ["content_rating"])
|
op.create_index("ix_plex_shows_content_rating", "plex_shows", ["content_rating"])
|
||||||
op.create_index("ix_plex_shows_studio", "plex_shows", ["studio"])
|
op.create_index("ix_plex_shows_studio", "plex_shows", ["studio"])
|
||||||
op.create_index("ix_plex_shows_originally_available_at", "plex_shows", ["originally_available_at"])
|
op.create_index(
|
||||||
|
"ix_plex_shows_originally_available_at", "plex_shows", ["originally_available_at"]
|
||||||
|
)
|
||||||
op.create_index("ix_plex_shows_genres_gin", "plex_shows", ["genres"], postgresql_using="gin")
|
op.create_index("ix_plex_shows_genres_gin", "plex_shows", ["genres"], postgresql_using="gin")
|
||||||
op.create_index("ix_plex_shows_directors_gin", "plex_shows", ["directors"], postgresql_using="gin")
|
op.create_index(
|
||||||
|
"ix_plex_shows_directors_gin", "plex_shows", ["directors"], postgresql_using="gin"
|
||||||
|
)
|
||||||
op.create_index("ix_plex_shows_cast_gin", "plex_shows", ["cast_names"], postgresql_using="gin")
|
op.create_index("ix_plex_shows_cast_gin", "plex_shows", ["cast_names"], postgresql_using="gin")
|
||||||
# NB: ix_plex_shows_collections_gin already exists (created with collections in 0047).
|
# NB: ix_plex_shows_collections_gin already exists (created with collections in 0047).
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ current_user rejects any cookie whose epoch is stale. Bumped on password reset,
|
|||||||
or "log out everywhere" — so a stolen/copied client-side cookie dies on those events instead of
|
or "log out everywhere" — so a stolen/copied client-side cookie dies on those events instead of
|
||||||
staying valid until it expires. Existing rows default to 0 (matching a fresh cookie's absent/0 epoch).
|
staying valid until it expires. Existing rows default to 0 (matching a fresh cookie's absent/0 epoch).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0054_audit_log
|
|||||||
Revises: 0053_user_session_epoch
|
Revises: 0053_user_session_epoch
|
||||||
Create Date: 2026-07-12
|
Create Date: 2026-07-12
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0055_channel_keywords
|
|||||||
Revises: 0054_audit_log
|
Revises: 0054_audit_log
|
||||||
Create Date: 2026-07-12
|
Create Date: 2026-07-12
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0056_plex_people
|
|||||||
Revises: 0055_channel_keywords
|
Revises: 0055_channel_keywords
|
||||||
Create Date: 2026-07-19
|
Create Date: 2026-07-19
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -23,8 +24,12 @@ def upgrade() -> None:
|
|||||||
# A representative proxied headshot URL (or NULL when none is available). Captured
|
# A representative proxied headshot URL (or NULL when none is available). Captured
|
||||||
# opportunistically from an item's rich metadata (info-page view + the rich re-sync).
|
# opportunistically from an item's rich metadata (info-page view + the rich re-sync).
|
||||||
sa.Column("photo_thumb", sa.Text(), nullable=True),
|
sa.Column("photo_thumb", sa.Text(), nullable=True),
|
||||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
sa.Column(
|
||||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
)
|
)
|
||||||
op.create_index("ix_plex_people_name", "plex_people", ["name"], unique=True)
|
op.create_index("ix_plex_people_name", "plex_people", ["name"], unique=True)
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Revision ID: 0057_plex_ppl_enriched
|
|||||||
Revises: 0056_plex_people
|
Revises: 0056_plex_people
|
||||||
Create Date: 2026-07-19
|
Create Date: 2026-07-19
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -17,7 +18,9 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
for tbl in ("plex_items", "plex_shows"):
|
for tbl in ("plex_items", "plex_shows"):
|
||||||
op.add_column(tbl, sa.Column("people_enriched_at", sa.DateTime(timezone=True), nullable=True))
|
op.add_column(
|
||||||
|
tbl, sa.Column("people_enriched_at", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ Best still one click away. Re-seeds the builtin rows so fresh installs and exist
|
|||||||
(same mechanism as 0040). Existing jobs are unaffected (they snapshot their spec; a profile_id that
|
(same mechanism as 0040). Existing jobs are unaffected (they snapshot their spec; a profile_id that
|
||||||
pointed at a replaced row just SET NULLs).
|
pointed at a replaced row just SET NULLs).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ did NOT invalidate outstanding grants — they lived out their 6h TTL. Fold a pe
|
|||||||
into the signed grant and bump it on every password change, so a rotation revokes old grants at once.
|
into the signed grant and bump it on every password change, so a rotation revokes old grants at once.
|
||||||
Small table (one row per share link), so this is a quick add-column with a default.
|
Small table (one row per share link), so this is a quick add-column with a default.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ This one-time data fixup rewrites those scalars to SQL NULL so the columns are u
|
|||||||
forward-only: a SQL NULL is indistinguishable from the old JSON `null` to every reader (the guards
|
forward-only: a SQL NULL is indistinguishable from the old JSON `null` to every reader (the guards
|
||||||
treat both as "not an array"), so there is nothing meaningful to downgrade. No schema change.
|
treat both as "not an array"), so there is nothing meaningful to downgrade. No schema change.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
@@ -25,9 +26,7 @@ _COLS = ("genres", "directors", "cast_names", "collection_keys")
|
|||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
for table in _TABLES:
|
for table in _TABLES:
|
||||||
for col in _COLS:
|
for col in _COLS:
|
||||||
op.execute(
|
op.execute(f"UPDATE {table} SET {col} = NULL WHERE jsonb_typeof({col}) = 'null'")
|
||||||
f"UPDATE {table} SET {col} = NULL WHERE jsonb_typeof({col}) = 'null'"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ small category code (`unavailable` / `login_required` / `geo_blocked` / `format_
|
|||||||
a retry makes sense, keeping the raw text as a technical-detail disclosure. Nullable; only set on the
|
a retry makes sense, keeping the raw text as a technical-detail disclosure. Nullable; only set on the
|
||||||
error path. Forward-only — an existing errored row simply has NULL until its next failure/retry.
|
error path. Forward-only — an existing errored row simply has NULL until its next failure/retry.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ Redundant index -> the composite that already covers it:
|
|||||||
ix_plex_states_user_id <- uq_plex_user_item (user_id, item_id)
|
ix_plex_states_user_id <- uq_plex_user_item (user_id, item_id)
|
||||||
ix_plex_playlist_items_playlist_id <- uq_plex_playlist_item (playlist_id, item_id)
|
ix_plex_playlist_items_playlist_id <- uq_plex_playlist_item (playlist_id, item_id)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ OVERLAY library is the Hungarian-metadata twin of another section: it is never b
|
|||||||
its titles' HU fields are written onto the matched EN rows (paired by `guid`, see 0064). Cheap:
|
its titles' HU fields are written onto the matched EN rows (paired by `guid`, see 0064). Cheap:
|
||||||
metadata-only, no table rewrite.
|
metadata-only, no table rewrite.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ Run BEFORE the first HU overlay sync: while the *_hu columns are all NULL the ge
|
|||||||
is trivial. ADD only (no drop/rebuild of the EN search_vector). NOTE: adding a STORED generated
|
is trivial. ADD only (no drop/rebuild of the EN search_vector). NOTE: adding a STORED generated
|
||||||
column + its GIN takes an ACCESS EXCLUSIVE lock and rewrites the table — brief, but flag it on prod.
|
column + its GIN takes an ACCESS EXCLUSIVE lock and rewrites the table — brief, but flag it on prod.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ so the data step below resets `people_enriched_at` to NULL on movie/show rows: t
|
|||||||
each once (60/run, self-throttled) and stamps the new columns. ADD-only, all nullable → no table
|
each once (60/run, self-throttled) and stamps the new columns. ADD-only, all nullable → no table
|
||||||
rewrite, no generated column, no ACCESS EXCLUSIVE lock (unlike 0064).
|
rewrite, no generated column, no ACCESS EXCLUSIVE lock (unlike 0064).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ schema allowed NULL — a model/schema drift the new migration drift-test (E1 S2
|
|||||||
server_default has filled every row since day one, so no NULLs exist; enforcing NOT NULL just makes the
|
server_default has filled every row since day one, so no NULLs exist; enforcing NOT NULL just makes the
|
||||||
schema match the contract the model already promises. Pure DDL, reversible.
|
schema match the contract the model already promises. Pure DDL, reversible.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ CONCURRENTLY (in alembic's autocommit_block, since it can't run inside a transac
|
|||||||
these without blocking reads/writes; downtime ~0. IF (NOT) EXISTS keeps the drift test's
|
these without blocking reads/writes; downtime ~0. IF (NOT) EXISTS keeps the drift test's
|
||||||
downgrade->re-upgrade round-trip idempotent.
|
downgrade->re-upgrade round-trip idempotent.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ Note: `SET NOT NULL` validates with a brief ACCESS EXCLUSIVE lock + one table sc
|
|||||||
CONCURRENTLY). On the prod row count (~235k) that is sub-second; still, it is a short write-blocking lock
|
CONCURRENTLY). On the prod row count (~235k) that is sub-second; still, it is a short write-blocking lock
|
||||||
rather than the ~0 of 0067's CONCURRENTLY index build.
|
rather than the ~0 of 0067's CONCURRENTLY index build.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
@@ -27,9 +28,7 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
# Backfill any stray NULLs first so SET NOT NULL can't fail (0 in prod, but be safe).
|
# Backfill any stray NULLs first so SET NOT NULL can't fail (0 in prod, but be safe).
|
||||||
op.execute("UPDATE videos SET published_at = now() WHERE published_at IS NULL")
|
op.execute("UPDATE videos SET published_at = now() WHERE published_at IS NULL")
|
||||||
op.alter_column(
|
op.alter_column("videos", "published_at", nullable=False, server_default=sa.text("now()"))
|
||||||
"videos", "published_at", nullable=False, server_default=sa.text("now()")
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ biggest table, same lookups. live_status is left alone (a separate finding).
|
|||||||
All CONCURRENTLY (autocommit_block) — downtime ~0; IF (NOT) EXISTS keeps the drift test's
|
All CONCURRENTLY (autocommit_block) — downtime ~0; IF (NOT) EXISTS keeps the drift test's
|
||||||
downgrade->re-upgrade idempotent.
|
downgrade->re-upgrade idempotent.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ hand-rolled override-or-default reader in `app.state`, duplicating the sysconfig
|
|||||||
now a sysconfig key (`system_config` store), so every admin knob lives in one place. This migration
|
now a sysconfig key (`system_config` store), so every admin knob lives in one place. This migration
|
||||||
carries any existing override into the sysconfig store, then drops the dead column. Reversible.
|
carries any existing override into the sysconfig store, then drops the dead column. Reversible.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Plex-sourced `imdb_rating`/`rating`, which are left untouched. Plus `plex_people
|
|||||||
foundation for S6's person bios). ADD-only, all nullable → no table rewrite, no ACCESS EXCLUSIVE lock.
|
foundation for S6's person bios). ADD-only, all nullable → no table rewrite, no ACCESS EXCLUSIVE lock.
|
||||||
No watermark reset needed: `tmdb_enriched_at` starts NULL, so every eligible row is picked up once.
|
No watermark reset needed: `tmdb_enriched_at` starts NULL, so every eligible row is picked up once.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ first time a person page is opened (from /person/{tmdb_person_id}), then SERVED
|
|||||||
`tmdb_person_id` link itself already exists (0071). ADD-only, all nullable → no table rewrite, no
|
`tmdb_person_id` link itself already exists (0071). ADD-only, all nullable → no table rewrite, no
|
||||||
ACCESS EXCLUSIVE lock. `tmdb_enriched_at` starts NULL, so every person is enriched once on first view.
|
ACCESS EXCLUSIVE lock. `tmdb_enriched_at` starts NULL, so every person is enriched once on first view.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""saved_views.source (ask 4: per-module saved views)
|
||||||
|
|
||||||
|
Saved "smart views" were feed-only; the Plex module now has them too. Add a `source` discriminator
|
||||||
|
(default "feed" so every existing row stays a feed view) and widen the per-user unique name constraint
|
||||||
|
to (user_id, source, name) so a feed view and a Plex view can share a name. `source` is NOT NULL with a
|
||||||
|
server_default, so the ADD is a metadata-only rewrite-free change; the constraint swap is quick on a
|
||||||
|
small per-user table.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0073_saved_view_source"
|
||||||
|
down_revision: Union[str, None] = "0072_plex_person_bio"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"saved_views",
|
||||||
|
sa.Column("source", sa.String(length=16), nullable=False, server_default="feed"),
|
||||||
|
)
|
||||||
|
op.drop_constraint("uq_user_saved_view", "saved_views", type_="unique")
|
||||||
|
op.create_unique_constraint(
|
||||||
|
"uq_user_source_saved_view", "saved_views", ["user_id", "source", "name"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# LOSSY / effectively forward-only: upgrade lets a user hold a feed view and a plex view of the same
|
||||||
|
# name, so re-creating the narrow (user_id, name) constraint can raise IntegrityError on such a pair.
|
||||||
|
# De-dup by dropping the non-feed rows first (they can't exist in the pre-0073 schema anyway) so the
|
||||||
|
# narrow constraint re-applies cleanly for the drift test's downgrade→re-upgrade cycle.
|
||||||
|
op.execute("DELETE FROM saved_views WHERE source <> 'feed'")
|
||||||
|
op.drop_constraint("uq_user_source_saved_view", "saved_views", type_="unique")
|
||||||
|
op.create_unique_constraint("uq_user_saved_view", "saved_views", ["user_id", "name"])
|
||||||
|
op.drop_column("saved_views", "source")
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""download_credentials (S6: per-user, per-host yt-dlp cookie/auth store)
|
||||||
|
|
||||||
|
Each user stores their OWN login cookies (Netscape cookies.txt) for an auth-walled download host
|
||||||
|
(Facebook, Instagram, …), Fernet-encrypted at rest. A download of a yt-dlp-supported, auth-walled
|
||||||
|
link on that host is retried with the acting user's cookie. Unique per (user_id, host_key) so a
|
||||||
|
user holds at most one credential per host. Empty for every existing install; forward-only in
|
||||||
|
practice (a downgrade just drops the table + its secrets).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0074_download_credentials"
|
||||||
|
down_revision: Union[str, None] = "0073_saved_view_source"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"download_credentials",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("host_key", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("cookies_enc", sa.Text(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint("user_id", "host_key", name="uq_download_credential"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_download_credentials_user_id", "download_credentials", ["user_id"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_download_credentials_user_id", table_name="download_credentials")
|
||||||
|
op.drop_table("download_credentials")
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
"""playlist synced_video_ids (S7: 3-way merge baseline for two-way YouTube sync)
|
||||||
|
|
||||||
|
The one-directional "local wins" push is replaced by a 3-way merge that classifies each item
|
||||||
|
against the last-synced state. That needs the raw ordered id set as a baseline, not just the
|
||||||
|
`synced_fingerprint` hash. Add a nullable `synced_video_ids` (JSON list) column; NULL on every
|
||||||
|
existing row means "no baseline yet" — the first merge then unions losslessly and records one.
|
||||||
|
Forward-only in practice (a downgrade just drops the column).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0075_playlist_sync_baseline"
|
||||||
|
down_revision: Union[str, None] = "0074_download_credentials"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("playlists", sa.Column("synced_video_ids", sa.Text(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("playlists", "synced_video_ids")
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
"""media_assets.media_kind (E7 S5a: the video|audio|file discriminator for the media library)
|
||||||
|
|
||||||
|
Adds a stored media-kind to each asset so the library can gate its actions on it (play/edit for
|
||||||
|
media, download-only for a generic uploaded file) instead of re-deriving from codecs in prose. The
|
||||||
|
column is the single home of the invariant `downloads.mediakind.derive` computes at finalize.
|
||||||
|
|
||||||
|
Backfill for existing rows: a stored video codec -> 'video', an audio-only codec -> 'audio', an
|
||||||
|
editor derivative (source_kind='edit', which carries no codecs but is always a video clip) ->
|
||||||
|
'video'. Rows still pending/errored (no codecs, not an edit) stay NULL — kind is set when they
|
||||||
|
finalize. Empty `media_kind` never reaches the UI: only `ready`/`done` rows render actions, and
|
||||||
|
those are all backfilled here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0076_media_asset_kind"
|
||||||
|
down_revision: Union[str, None] = "0075_playlist_sync_baseline"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("media_assets", sa.Column("media_kind", sa.String(length=16), nullable=True))
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE media_assets
|
||||||
|
SET media_kind = CASE
|
||||||
|
WHEN vcodec IS NOT NULL THEN 'video'
|
||||||
|
WHEN acodec IS NOT NULL THEN 'audio'
|
||||||
|
WHEN source_kind = 'edit' THEN 'video'
|
||||||
|
ELSE NULL
|
||||||
|
END
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("media_assets", "media_kind")
|
||||||
@@ -9,6 +9,7 @@ runs are always logged. Secrets are never stored (config changes log the key nam
|
|||||||
mutation it describes, so the trail can't drift from reality (and a rolled-back mutation drops
|
mutation it describes, so the trail can't drift from reality (and a rolled-back mutation drops
|
||||||
its audit row too).
|
its audit row too).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
from sqlalchemy import delete
|
from sqlalchemy import delete
|
||||||
@@ -43,6 +44,9 @@ class AuditAction:
|
|||||||
SYNC_RESUME = "sync_resume"
|
SYNC_RESUME = "sync_resume"
|
||||||
# Scheduler job outcome / run-summary (actor_id set = a manual "run now"; NULL = automatic)
|
# Scheduler job outcome / run-summary (actor_id set = a manual "run now"; NULL = automatic)
|
||||||
JOB_RUN = "job_run"
|
JOB_RUN = "job_run"
|
||||||
|
# Per-user download cookie/auth credential set or removed (the value is NEVER recorded)
|
||||||
|
DOWNLOAD_CREDENTIAL_SET = "download_credential_set"
|
||||||
|
DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete"
|
||||||
# The log itself was cleared (a single row survives the clear as tamper-evidence)
|
# The log itself was cleared (a single row survives the clear as tamper-evidence)
|
||||||
AUDIT_CLEAR = "audit_clear"
|
AUDIT_CLEAR = "audit_clear"
|
||||||
|
|
||||||
|
|||||||
+6
-9
@@ -41,8 +41,7 @@ log = logging.getLogger("siftlode.auth")
|
|||||||
def google_enabled(db: Session) -> bool:
|
def google_enabled(db: Session) -> bool:
|
||||||
"""Whether Sign in with Google is available (both client id + secret resolve to non-empty)."""
|
"""Whether Sign in with Google is available (both client id + secret resolve to non-empty)."""
|
||||||
return bool(
|
return bool(
|
||||||
sysconfig.get_str(db, "google_client_id")
|
sysconfig.get_str(db, "google_client_id") and sysconfig.get_str(db, "google_client_secret")
|
||||||
and sysconfig.get_str(db, "google_client_secret")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -125,7 +124,9 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
|||||||
# ciphertext, so a token row with no refresh token silently failed to revoke). decrypt_optional so a
|
# ciphertext, so a token row with no refresh token silently failed to revoke). decrypt_optional so a
|
||||||
# TOKEN_ENCRYPTION_KEY mismatch can't 500 the purge — a best-effort revoke degrades to "skip". (B-core-2)
|
# TOKEN_ENCRYPTION_KEY mismatch can't 500 the purge — a best-effort revoke degrades to "skip". (B-core-2)
|
||||||
google_token = (
|
google_token = (
|
||||||
(decrypt_optional(tok.refresh_token_enc) or decrypt_optional(tok.access_token)) if tok else None
|
(decrypt_optional(tok.refresh_token_enc) or decrypt_optional(tok.access_token))
|
||||||
|
if tok
|
||||||
|
else None
|
||||||
)
|
)
|
||||||
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
|
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
|
||||||
db.execute(delete(User).where(User.id == user_id))
|
db.execute(delete(User).where(User.id == user_id))
|
||||||
@@ -197,9 +198,7 @@ def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
|||||||
return user
|
return user
|
||||||
|
|
||||||
|
|
||||||
def optional_current_user(
|
def optional_current_user(request: Request, db: Session = Depends(get_db)) -> User | None:
|
||||||
request: Request, db: Session = Depends(get_db)
|
|
||||||
) -> User | None:
|
|
||||||
"""Like `current_user` but returns None instead of raising 401 when there's no valid session.
|
"""Like `current_user` but returns None instead of raising 401 when there's no valid session.
|
||||||
Used by the bootstrap /api/me probe so the logged-out landing doesn't log a failed request to
|
Used by the bootstrap /api/me probe so the logged-out landing doesn't log a failed request to
|
||||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it)."""
|
the browser console (a non-2xx fetch is a console error no matter how the app handles it)."""
|
||||||
@@ -215,9 +214,7 @@ def require_human(user: User = Depends(current_user)) -> User:
|
|||||||
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
|
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
|
||||||
where there's no scope check to lean on."""
|
where there's no scope check to lean on."""
|
||||||
if user.is_demo:
|
if user.is_demo:
|
||||||
raise HTTPException(
|
raise HTTPException(status_code=403, detail="Not available in the demo account.")
|
||||||
status_code=403, detail="Not available in the demo account."
|
|
||||||
)
|
|
||||||
return user
|
return user
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+23
-1
@@ -169,6 +169,7 @@ class Settings(BaseSettings):
|
|||||||
@property
|
@property
|
||||||
def player_client_fallback_list(self) -> list[str]:
|
def player_client_fallback_list(self) -> list[str]:
|
||||||
return self._clients(self.download_player_clients_fallback)
|
return self._clients(self.download_player_clients_fallback)
|
||||||
|
|
||||||
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
|
# How many downloads the worker runs concurrently (claimed via FOR UPDATE SKIP LOCKED).
|
||||||
download_worker_concurrency: int = 2
|
download_worker_concurrency: int = 2
|
||||||
# How often (empty pending queue) the worker polls for a new job, in seconds.
|
# How often (empty pending queue) the worker polls for a new job, in seconds.
|
||||||
@@ -185,6 +186,21 @@ class Settings(BaseSettings):
|
|||||||
download_supervisor_poll_seconds: int = 3
|
download_supervisor_poll_seconds: int = 3
|
||||||
# Grace between SIGTERM and SIGKILL when stopping a child (matches run_ffmpeg's escalation).
|
# Grace between SIGTERM and SIGKILL when stopping a child (matches run_ffmpeg's escalation).
|
||||||
download_kill_grace_seconds: int = 5
|
download_kill_grace_seconds: int = 5
|
||||||
|
# --- worker-child resource de-prioritization ---
|
||||||
|
# The download/edit child (yt-dlp + ffmpeg) shares ONE LAN box with the API, Postgres, and Plex
|
||||||
|
# playback. Without a priority handicap a CPU re-encode pegs every core for ~30s and starves web
|
||||||
|
# requests + playback until it finishes. The child lowers its OWN priority at startup — inherited
|
||||||
|
# by every ffmpeg/yt-dlp descendant — so any foreground work always wins. Each knob is best-effort
|
||||||
|
# and independently disable-able with 0.
|
||||||
|
download_child_nice: int = 10 # CPU nice increment applied to the child; 0 = don't renice
|
||||||
|
download_child_ionice_class: int = 3 # best-effort IO class (3 = idle); 0 = leave IO untouched
|
||||||
|
# Cap the browser-safe re-encode's encoder threads so a transcode can't claim every core. 0 =
|
||||||
|
# reserve one core for foreground work (max(1, cpu_count - 1)); >0 pins exactly that many.
|
||||||
|
download_ffmpeg_threads: int = 0
|
||||||
|
# Throttle the retention GC's whole-tree orphan file sweep so its rglob can't monopolize disk IO
|
||||||
|
# in the API process: sleep `pause_ms` after every `batch` files touched. pause_ms=0 disables.
|
||||||
|
download_gc_sweep_pause_ms: int = 20
|
||||||
|
download_gc_sweep_batch: int = 400
|
||||||
# Total cache size cap across ALL users (bytes). When >0 and exceeded, the GC evicts the
|
# Total cache size cap across ALL users (bytes). When >0 and exceeded, the GC evicts the
|
||||||
# least-recently-accessed ready assets until under it. 0 = unlimited (disk is the limit).
|
# least-recently-accessed ready assets until under it. 0 = unlimited (disk is the limit).
|
||||||
download_total_max_bytes: int = 0
|
download_total_max_bytes: int = 0
|
||||||
@@ -192,6 +208,10 @@ class Settings(BaseSettings):
|
|||||||
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
|
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
|
||||||
download_default_max_concurrent: int = 2
|
download_default_max_concurrent: int = 2
|
||||||
download_default_max_jobs: int = 50
|
download_default_max_jobs: int = 50
|
||||||
|
# Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker
|
||||||
|
# finishes), an upload's size is known at ingest, so this bounds a single file directly; the
|
||||||
|
# user's overall byte footprint is still governed by download_default_max_bytes.
|
||||||
|
download_upload_max_bytes: int = 524_288_000 # 500 MiB
|
||||||
# Retention: an asset expires this many days after its last access; the GC job then
|
# Retention: an asset expires this many days after its last access; the GC job then
|
||||||
# deletes it (once no job references it) after an additional grace window.
|
# deletes it (once no job references it) after an additional grace window.
|
||||||
download_retention_days: int = 30
|
download_retention_days: int = 30
|
||||||
@@ -231,7 +251,9 @@ class Settings(BaseSettings):
|
|||||||
plex_sync_interval_min: int = 360
|
plex_sync_interval_min: int = 360
|
||||||
# How often the incremental Plex→Siftlode watch-state reconcile runs, in minutes (Phase C):
|
# How often the incremental Plex→Siftlode watch-state reconcile runs, in minutes (Phase C):
|
||||||
# a cheap history + on-deck pull plus a re-push of any states that failed their immediate push.
|
# a cheap history + on-deck pull plus a re-push of any states that failed their immediate push.
|
||||||
plex_watch_sync_interval_min: int = 30
|
# 15 min (S2): with Plex now the source of truth (prompt write-back on the ~10s checkpoint +
|
||||||
|
# Plex-authoritative pull), a faster incremental pull keeps Siftlode close to Plex-side changes.
|
||||||
|
plex_watch_sync_interval_min: int = 15
|
||||||
# How often the FULL watch-state reconcile runs, in minutes (Phase C). A whole-section rescan
|
# How often the FULL watch-state reconcile runs, in minutes (Phase C). A whole-section rescan
|
||||||
# that catches what the incremental history feed can't — notably an un-watch done on the Plex
|
# that catches what the incremental history feed can't — notably an un-watch done on the Plex
|
||||||
# side — so it runs far less often (daily) than the incremental pass.
|
# side — so it runs far less often (daily) than the incremental pass.
|
||||||
|
|||||||
+12
-7
@@ -4,6 +4,7 @@ Shared by the admin's manual reset endpoint (routes/admin.py) and the scheduled
|
|||||||
(scheduler.py). Lives in the service layer, not routes, so the scheduler can call it without an
|
(scheduler.py). Lives in the service layer, not routes, so the scheduler can call it without an
|
||||||
`app.routes -> app.scheduler` import cycle (the reason its import used to be lazy).
|
`app.routes -> app.scheduler` import cycle (the reason its import used to be lazy).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from sqlalchemy import delete, func, select
|
from sqlalchemy import delete, func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
@@ -49,13 +50,17 @@ def _seed_demo_subscriptions(db: Session, demo: User, n: int = 14) -> int:
|
|||||||
Channel manager isn't empty — gives visitors something to explore and matches the landing-page
|
Channel manager isn't empty — gives visitors something to explore and matches the landing-page
|
||||||
screenshot. Idempotent: clears the demo's subscriptions first, then re-adds the top N."""
|
screenshot. Idempotent: clears the demo's subscriptions first, then re-adds the top N."""
|
||||||
db.execute(delete(Subscription).where(Subscription.user_id == demo.id))
|
db.execute(delete(Subscription).where(Subscription.user_id == demo.id))
|
||||||
top = db.execute(
|
top = (
|
||||||
select(Video.channel_id)
|
db.execute(
|
||||||
.where(Video.channel_id.is_not(None))
|
select(Video.channel_id)
|
||||||
.group_by(Video.channel_id)
|
.where(Video.channel_id.is_not(None))
|
||||||
.order_by(func.count().desc())
|
.group_by(Video.channel_id)
|
||||||
.limit(n)
|
.order_by(func.count().desc())
|
||||||
).scalars().all()
|
.limit(n)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for channel_id in top:
|
for channel_id in top:
|
||||||
db.add(Subscription(user_id=demo.id, channel_id=channel_id))
|
db.add(Subscription(user_id=demo.id, channel_id=channel_id))
|
||||||
return len(top)
|
return len(top)
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ Adapters are stateless singletons; get one with `resolve_adapter(source_kind, so
|
|||||||
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
||||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
@@ -140,8 +141,10 @@ def resolve_adapter(
|
|||||||
def host_is(*domains: str) -> bool:
|
def host_is(*domains: str) -> bool:
|
||||||
return any(_host_is(h, *domains) for h in hosts)
|
return any(_host_is(h, *domains) for h in hosts)
|
||||||
|
|
||||||
if source_kind == "youtube" or "youtube" in extractor or host_is(
|
if (
|
||||||
"youtube.com", "youtu.be", "youtube-nocookie.com"
|
source_kind == "youtube"
|
||||||
|
or "youtube" in extractor
|
||||||
|
or host_is("youtube.com", "youtu.be", "youtube-nocookie.com")
|
||||||
):
|
):
|
||||||
return _YOUTUBE
|
return _YOUTUBE
|
||||||
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ URL may have expired — those just get a locally-generated poster instead).
|
|||||||
Run once per environment:
|
Run once per environment:
|
||||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
"""Per-user, per-host download cookies — the single source of truth for "does the acting user's
|
||||||
|
cookie apply to this download, and how does that keep the result private".
|
||||||
|
|
||||||
|
Two call sites depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||||
|
in prose):
|
||||||
|
* `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the
|
||||||
|
authed download becomes a PRIVATE asset that is never cache-served to another user.
|
||||||
|
* the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it
|
||||||
|
is downloading into is that private, salted identity — so a cookie can never poison a shared
|
||||||
|
asset (the load-bearing cross-user privacy fix).
|
||||||
|
|
||||||
|
Both go through `cookie_salt` / `authed_format_sig` / `host_key_for_url` here, so they can't drift.
|
||||||
|
|
||||||
|
Security controls enforced here:
|
||||||
|
* **host scope (SEC-12)** — `host_key_for_url` matches the download URL's netloc against a curated
|
||||||
|
registry via the netloc-scoped `_host_is`, so a cookie is only ever offered to the exact site it
|
||||||
|
belongs to; an arbitrary `?url=` (or a look-alike host) resolves to no host_key and gets no
|
||||||
|
cookie.
|
||||||
|
* **secret handling** — cookies are Fernet-encrypted at rest; the blob is only ever decrypted
|
||||||
|
in-worker into a 0600 file that is unlinked in a `finally`, and is never returned by a route,
|
||||||
|
logged, or written to a job's error/snapshot.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.downloads import formats
|
||||||
|
from app.downloads.adapters import _host_is
|
||||||
|
from app.models import DownloadCredential
|
||||||
|
from app.security import decrypt_optional, encrypt
|
||||||
|
|
||||||
|
log = logging.getLogger("siftlode.downloads.credentials")
|
||||||
|
|
||||||
|
|
||||||
|
class HostSpec:
|
||||||
|
"""A supported cookie host: a short canonical `key` (stored on the credential + folded into the
|
||||||
|
cache salt), a human `label` for the UI, and the netloc `domains` it matches."""
|
||||||
|
|
||||||
|
__slots__ = ("key", "label", "domains")
|
||||||
|
|
||||||
|
def __init__(self, key: str, label: str, domains: tuple[str, ...]):
|
||||||
|
self.key = key
|
||||||
|
self.label = label
|
||||||
|
self.domains = domains
|
||||||
|
|
||||||
|
|
||||||
|
# Curated registry (user's call: seed the common auth-walled hosts now). One `key` can cover several
|
||||||
|
# domains of the same site. Add a host by adding a row — the FE dropdown, host-scope guard, and salt
|
||||||
|
# all derive from this, so there is nothing else to wire up.
|
||||||
|
SUPPORTED_HOSTS: tuple[HostSpec, ...] = (
|
||||||
|
HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")),
|
||||||
|
HostSpec("instagram", "Instagram", ("instagram.com",)),
|
||||||
|
HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")),
|
||||||
|
HostSpec("tiktok", "TikTok", ("tiktok.com",)),
|
||||||
|
HostSpec("reddit", "Reddit", ("reddit.com",)),
|
||||||
|
)
|
||||||
|
|
||||||
|
_BY_KEY: dict[str, HostSpec] = {h.key: h for h in SUPPORTED_HOSTS}
|
||||||
|
|
||||||
|
|
||||||
|
def is_supported_host(host_key: str) -> bool:
|
||||||
|
return host_key in _BY_KEY
|
||||||
|
|
||||||
|
|
||||||
|
def host_label(host_key: str) -> str:
|
||||||
|
spec = _BY_KEY.get(host_key)
|
||||||
|
return spec.label if spec else host_key
|
||||||
|
|
||||||
|
|
||||||
|
def host_key_for_url(url: str | None) -> str | None:
|
||||||
|
"""The supported host_key whose domains own `url`'s netloc, or None. Netloc-scoped (via
|
||||||
|
`_host_is`), so a look-alike host ('myfacebook.com', 'facebook.com.evil') never matches. When
|
||||||
|
several specs could match, the most-specific (longest matched domain) wins so the result is
|
||||||
|
deterministic across the enqueue and worker call sites."""
|
||||||
|
if not url:
|
||||||
|
return None
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
host = urlparse(url).netloc.lower()
|
||||||
|
if not host:
|
||||||
|
return None
|
||||||
|
best_key: str | None = None
|
||||||
|
best_len = -1
|
||||||
|
for spec in SUPPORTED_HOSTS:
|
||||||
|
for dom in spec.domains:
|
||||||
|
if _host_is(host, dom) and len(dom) > best_len:
|
||||||
|
best_key, best_len = spec.key, len(dom)
|
||||||
|
return best_key
|
||||||
|
|
||||||
|
|
||||||
|
def cookie_salt(user_id: int, host_key: str) -> str:
|
||||||
|
"""The cache-signature salt for a cookie-authed download. Keyed by user id + host so the authed
|
||||||
|
asset is private to that user (never dedup-shared) and stable across that user's own re-adds."""
|
||||||
|
return f"cookie:{user_id}:{host_key}"
|
||||||
|
|
||||||
|
|
||||||
|
def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str:
|
||||||
|
"""The salted `format_sig` an authed download for (user, host) resolves to — the private asset
|
||||||
|
identity. Both enqueue (to create it) and the worker (to recognise it) compute it HERE."""
|
||||||
|
return formats.format_sig(spec, salt=cookie_salt(user_id, host_key))
|
||||||
|
|
||||||
|
|
||||||
|
def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
||||||
|
"""At enqueue: the cache salt to apply if this user has a cookie for `url`'s host, else None (a
|
||||||
|
normal shared download). Returns the salt string so the caller passes it straight to
|
||||||
|
`format_sig(spec, salt=...)`."""
|
||||||
|
host_key = host_key_for_url(url)
|
||||||
|
if host_key is None:
|
||||||
|
return None
|
||||||
|
if get_credential(db, user_id, host_key) is None:
|
||||||
|
return None
|
||||||
|
return cookie_salt(user_id, host_key)
|
||||||
|
|
||||||
|
|
||||||
|
# --- storage (CRUD) ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None:
|
||||||
|
return db.execute(
|
||||||
|
select(DownloadCredential).where(
|
||||||
|
DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidCookies(ValueError):
|
||||||
|
"""The pasted cookies aren't a usable Netscape cookies.txt — surfaced as a 400 by the route."""
|
||||||
|
|
||||||
|
|
||||||
|
def validate_cookies(raw: str) -> str:
|
||||||
|
"""Sanity-check a pasted cookies.txt (Netscape format) and return the cleaned text. yt-dlp reads
|
||||||
|
the Netscape format: comment/`#` lines plus TAB-separated cookie lines (domain, flag, path,
|
||||||
|
secure, expiry, name, value). We don't fully parse — just reject obvious non-cookie paste (empty,
|
||||||
|
or no tab-separated data line) so a mistaken paste fails HERE with a clear message rather than
|
||||||
|
silently producing a cookie file yt-dlp ignores."""
|
||||||
|
text = (raw or "").strip()
|
||||||
|
if not text:
|
||||||
|
raise InvalidCookies("empty")
|
||||||
|
has_data_line = any(
|
||||||
|
line.strip() and not line.lstrip().startswith("#") and "\t" in line
|
||||||
|
for line in text.splitlines()
|
||||||
|
)
|
||||||
|
if not has_data_line:
|
||||||
|
raise InvalidCookies("format")
|
||||||
|
# Netscape cookie files must end with a newline for some parsers; normalise.
|
||||||
|
return text if text.endswith("\n") else text + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_credential(db: Session, user_id: int, host_key: str, raw: str) -> DownloadCredential:
|
||||||
|
"""Create or replace this user's cookie blob for `host_key`. Caller has validated host_key is
|
||||||
|
supported and `raw` via `validate_cookies`. Does NOT commit (the route commits with the audit
|
||||||
|
row so the trail can't drift). The blob is encrypted here; the plaintext never leaves this call."""
|
||||||
|
cred = get_credential(db, user_id, host_key)
|
||||||
|
enc = encrypt(raw)
|
||||||
|
if cred is None:
|
||||||
|
cred = DownloadCredential(user_id=user_id, host_key=host_key, cookies_enc=enc)
|
||||||
|
db.add(cred)
|
||||||
|
else:
|
||||||
|
cred.cookies_enc = enc
|
||||||
|
return cred
|
||||||
|
|
||||||
|
|
||||||
|
def delete_credential(db: Session, user_id: int, host_key: str) -> bool:
|
||||||
|
"""Delete this user's cookie for `host_key`. Returns whether a row existed. Does NOT commit."""
|
||||||
|
cred = get_credential(db, user_id, host_key)
|
||||||
|
if cred is None:
|
||||||
|
return False
|
||||||
|
db.delete(cred)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def status_list(db: Session, user_id: int) -> list[dict]:
|
||||||
|
"""The supported hosts with this user's configured/updated state — the write-only Settings view.
|
||||||
|
Never includes the blob."""
|
||||||
|
have = {
|
||||||
|
c.host_key: c
|
||||||
|
for c in db.execute(
|
||||||
|
select(DownloadCredential).where(DownloadCredential.user_id == user_id)
|
||||||
|
).scalars()
|
||||||
|
}
|
||||||
|
out = []
|
||||||
|
for spec in SUPPORTED_HOSTS:
|
||||||
|
cred = have.get(spec.key)
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"host_key": spec.key,
|
||||||
|
"label": spec.label,
|
||||||
|
"configured": cred is not None,
|
||||||
|
"updated_at": cred.updated_at.isoformat() if cred is not None else None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# --- worker: resolve + write the cookie file --------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_cookiefile(
|
||||||
|
db: Session,
|
||||||
|
user_id: int,
|
||||||
|
url: str | None,
|
||||||
|
asset_format_sig: str,
|
||||||
|
spec: dict,
|
||||||
|
staging: Path,
|
||||||
|
) -> Path | None:
|
||||||
|
"""In the worker child: the cookies.txt to hand yt-dlp for this download, or None.
|
||||||
|
|
||||||
|
The double gate that prevents cross-user leakage:
|
||||||
|
1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND
|
||||||
|
2. the asset being downloaded into must BE this user's private, salted identity
|
||||||
|
(`asset_format_sig == authed_format_sig(...)`).
|
||||||
|
|
||||||
|
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
||||||
|
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
||||||
|
InvalidToken) also yields None; the download then fails honestly on the login wall (S3's
|
||||||
|
`login_required`)."""
|
||||||
|
host_key = host_key_for_url(url)
|
||||||
|
if host_key is None:
|
||||||
|
return None
|
||||||
|
cred = get_credential(db, user_id, host_key)
|
||||||
|
if cred is None:
|
||||||
|
return None
|
||||||
|
if asset_format_sig != authed_format_sig(spec, user_id, host_key):
|
||||||
|
return None # shared asset — never attach a cookie to a non-private identity
|
||||||
|
raw = decrypt_optional(cred.cookies_enc)
|
||||||
|
if not raw:
|
||||||
|
log.warning(
|
||||||
|
"download cookie for user %s host %s could not be decrypted (rotated key?) — "
|
||||||
|
"proceeding without it",
|
||||||
|
user_id,
|
||||||
|
host_key,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
return _write_cookiefile(raw, staging)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
||||||
|
"""Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path.
|
||||||
|
|
||||||
|
Deliberately NOT in the staging root: the download retry loop unlinks every file it finds there,
|
||||||
|
and `_produced_file` picks the largest file there as the output — either would corrupt a
|
||||||
|
cookies.txt in the root. A `.auth` SUBDIR survives both (they skip directories) and is still
|
||||||
|
removed by the parent worker's staging cleanup on a hard kill. The caller unlinks it in a
|
||||||
|
`finally` on the normal/exception paths."""
|
||||||
|
auth_dir = staging / ".auth"
|
||||||
|
auth_dir.mkdir(mode=0o700, exist_ok=True)
|
||||||
|
path = auth_dir / "cookies.txt"
|
||||||
|
# Open with 0600 from the start (don't create world-readable then chmod).
|
||||||
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||||
|
try:
|
||||||
|
os.write(fd, raw.encode("utf-8"))
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
return path
|
||||||
+192
-46
@@ -20,10 +20,12 @@ Also builds the editor **filmstrip** — a single tiled sprite of the source vid
|
|||||||
scrub track. Cached on the source asset's `storyboard_path` (reserved in phase 1) and reusable
|
scrub track. Cached on the source asset's `storyboard_path` (reserved in phase 1) and reusable
|
||||||
for a player hover-preview later.
|
for a player hover-preview later.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import math
|
import math
|
||||||
|
import os
|
||||||
import queue
|
import queue
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
@@ -31,15 +33,28 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.edit")
|
log = logging.getLogger("siftlode.edit")
|
||||||
|
|
||||||
|
|
||||||
|
def encode_thread_cap() -> int:
|
||||||
|
"""How many encoder threads a browser-safe re-encode may use. `download_ffmpeg_threads` when the
|
||||||
|
admin pinned one (>0), else reserve a core for foreground work (max(1, cpu_count - 1)). Shared by
|
||||||
|
the ffmpeg `-threads` flag and the child's `OMP_NUM_THREADS` so both honour the same budget."""
|
||||||
|
n = settings.download_ffmpeg_threads
|
||||||
|
if n > 0:
|
||||||
|
return n
|
||||||
|
return max(1, (os.cpu_count() or 2) - 1)
|
||||||
|
|
||||||
|
|
||||||
class EditAborted(Exception):
|
class EditAborted(Exception):
|
||||||
"""Raised by run_ffmpeg when the job was paused/canceled mid-encode."""
|
"""Raised by run_ffmpeg when the job was paused/canceled mid-encode."""
|
||||||
|
|
||||||
|
|
||||||
# --- edit spec ------------------------------------------------------------------------------
|
# --- edit spec ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def normalize_edit_spec(spec: dict | None) -> dict:
|
def normalize_edit_spec(spec: dict | None) -> dict:
|
||||||
"""Canonicalize a raw edit spec: clamp/round trim, coerce crop to ints, drop empties.
|
"""Canonicalize a raw edit spec: clamp/round trim, coerce crop to ints, drop empties.
|
||||||
|
|
||||||
@@ -140,6 +155,7 @@ def clip_duration(spec: dict, source_duration: int | None) -> int | None:
|
|||||||
|
|
||||||
# --- ffmpeg: trim / crop --------------------------------------------------------------------
|
# --- ffmpeg: trim / crop --------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
||||||
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
|
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
|
||||||
|
|
||||||
@@ -153,8 +169,15 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
|||||||
reencode = needs_reencode(spec)
|
reencode = needs_reencode(spec)
|
||||||
|
|
||||||
cmd = [
|
cmd = [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-progress", "pipe:1", "-nostats",
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-progress",
|
||||||
|
"pipe:1",
|
||||||
|
"-nostats",
|
||||||
]
|
]
|
||||||
if start:
|
if start:
|
||||||
cmd += ["-ss", f"{start:.3f}"]
|
cmd += ["-ss", f"{start:.3f}"]
|
||||||
@@ -170,8 +193,16 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
|||||||
if crop:
|
if crop:
|
||||||
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
|
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
|
||||||
cmd += [
|
cmd += [
|
||||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
|
"-c:v",
|
||||||
"-c:a", "aac", "-b:a", "192k",
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"veryfast",
|
||||||
|
"-crf",
|
||||||
|
"20",
|
||||||
|
"-c:a",
|
||||||
|
"aac",
|
||||||
|
"-b:a",
|
||||||
|
"192k",
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
||||||
@@ -204,8 +235,15 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
|||||||
crop = spec.get("crop")
|
crop = spec.get("crop")
|
||||||
prep: dict[Path, str] = {}
|
prep: dict[Path, str] = {}
|
||||||
head = [
|
head = [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-progress", "pipe:1", "-nostats",
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-progress",
|
||||||
|
"pipe:1",
|
||||||
|
"-nostats",
|
||||||
]
|
]
|
||||||
|
|
||||||
if needs_reencode(spec):
|
if needs_reencode(spec):
|
||||||
@@ -218,9 +256,24 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
|||||||
labels.append(f"[v{i}][a{i}]")
|
labels.append(f"[v{i}][a{i}]")
|
||||||
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
||||||
cmd = head + [
|
cmd = head + [
|
||||||
"-i", str(src), "-filter_complex", ";".join(parts),
|
"-i",
|
||||||
"-map", "[v]", "-map", "[a]",
|
str(src),
|
||||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-c:a", "aac", "-b:a", "192k",
|
"-filter_complex",
|
||||||
|
";".join(parts),
|
||||||
|
"-map",
|
||||||
|
"[v]",
|
||||||
|
"-map",
|
||||||
|
"[a]",
|
||||||
|
"-c:v",
|
||||||
|
"libx264",
|
||||||
|
"-preset",
|
||||||
|
"veryfast",
|
||||||
|
"-crf",
|
||||||
|
"20",
|
||||||
|
"-c:a",
|
||||||
|
"aac",
|
||||||
|
"-b:a",
|
||||||
|
"192k",
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
listp = staging / "concat.txt"
|
listp = staging / "concat.txt"
|
||||||
@@ -314,7 +367,9 @@ def _stop(proc: subprocess.Popen) -> None:
|
|||||||
escalate_kill(proc, proc.terminate, proc.kill, grace_s=5, label="ffmpeg")
|
escalate_kill(proc, proc.terminate, proc.kill, grace_s=5, label="ffmpeg")
|
||||||
|
|
||||||
|
|
||||||
def run_ffmpeg(cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S) -> None:
|
def run_ffmpeg(
|
||||||
|
cmd, total_s, on_progress, should_cancel, stall_timeout_s=_FFMPEG_STALL_TIMEOUT_S
|
||||||
|
) -> None:
|
||||||
"""Run an ffmpeg edit, reporting 0–100 progress and honouring cooperative cancel.
|
"""Run an ffmpeg edit, reporting 0–100 progress and honouring cooperative cancel.
|
||||||
|
|
||||||
Both pipes are drained by threads (see `_drain_tail`) and the main loop polls on a 1s tick, so
|
Both pipes are drained by threads (see `_drain_tail`) and the main loop polls on a 1s tick, so
|
||||||
@@ -383,18 +438,42 @@ _BROWSER_SAFE_VCODEC = ("h264",)
|
|||||||
_BROWSER_SAFE_ACODEC = ("aac", "mp3")
|
_BROWSER_SAFE_ACODEC = ("aac", "mp3")
|
||||||
|
|
||||||
|
|
||||||
def parse_probe_streams(payload: str) -> tuple[str | None, str | None]:
|
def run_ffprobe_json(
|
||||||
"""(video_codec, audio_codec) out of ffprobe's JSON stream list, lowercased.
|
path: Path,
|
||||||
|
*,
|
||||||
JSON, not the flat `default=nw=1` output: there the fields arrive in ffprobe's own struct
|
show_streams: bool = False,
|
||||||
order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line "remember the last
|
show_format: bool = False,
|
||||||
codec_type" parser attributes every name to the PREVIOUS stream. Measured on real downloads:
|
entries: str | None = None,
|
||||||
an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
|
) -> dict | None:
|
||||||
believed the video stream was AAC and re-encoded a perfectly fine file on every download."""
|
"""One ffprobe call → its parsed JSON dict, or None on any failure (timeout, non-zero exit,
|
||||||
|
missing binary, unparseable output). The SINGLE ffprobe-JSON-invocation point in the downloads
|
||||||
|
package: the browser-playability codec probe here and the upload-ingest probe (`upload._probe`)
|
||||||
|
both go through it, so the subprocess flags/timeout/error handling live in one place instead of
|
||||||
|
two drifting copies."""
|
||||||
|
args = ["ffprobe", "-v", "error"]
|
||||||
|
if show_streams:
|
||||||
|
args.append("-show_streams")
|
||||||
|
if show_format:
|
||||||
|
args.append("-show_format")
|
||||||
|
if entries:
|
||||||
|
args += ["-show_entries", entries]
|
||||||
|
args += ["-of", "json", str(path)]
|
||||||
try:
|
try:
|
||||||
streams = json.loads(payload or "{}").get("streams") or []
|
out = subprocess.run(args, capture_output=True, text=True, timeout=30, check=True).stdout
|
||||||
except ValueError:
|
return json.loads(out)
|
||||||
return None, None
|
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _codecs_from_streams(streams: list) -> tuple[str | None, str | None]:
|
||||||
|
"""(video_codec, audio_codec) lowercased from an ffprobe JSON stream list — the first of each kind.
|
||||||
|
|
||||||
|
Parsed from the JSON `streams` list, NOT ffprobe's flat `default=nw=1` output: there the fields
|
||||||
|
arrive in ffprobe's own struct order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line
|
||||||
|
"remember the last codec_type" parser attributes every name to the PREVIOUS stream. Measured on real
|
||||||
|
downloads: an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
|
||||||
|
believed the video stream was AAC and re-encoded a perfectly fine file on every download. Reading each
|
||||||
|
stream dict by its own `codec_type` is immune to that ordering."""
|
||||||
vcodec = acodec = None
|
vcodec = acodec = None
|
||||||
for s in streams:
|
for s in streams:
|
||||||
name = (s.get("codec_name") or "").lower() or None
|
name = (s.get("codec_name") or "").lower() or None
|
||||||
@@ -409,15 +488,10 @@ def parse_probe_streams(payload: str) -> tuple[str | None, str | None]:
|
|||||||
def probe_codecs(path: Path) -> tuple[str | None, str | None]:
|
def probe_codecs(path: Path) -> tuple[str | None, str | None]:
|
||||||
"""Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing
|
"""Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing
|
||||||
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
|
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
|
||||||
try:
|
data = run_ffprobe_json(path, entries="stream=codec_type,codec_name")
|
||||||
out = subprocess.run(
|
if data is None:
|
||||||
["ffprobe", "-v", "error", "-show_entries", "stream=codec_type,codec_name",
|
|
||||||
"-of", "json", str(path)],
|
|
||||||
capture_output=True, text=True, timeout=30, check=True,
|
|
||||||
).stdout
|
|
||||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
|
|
||||||
return None, None
|
return None, None
|
||||||
return parse_probe_streams(out)
|
return _codecs_from_streams(data.get("streams") or [])
|
||||||
|
|
||||||
|
|
||||||
def probe_duration(path: Path) -> int:
|
def probe_duration(path: Path) -> int:
|
||||||
@@ -425,16 +499,29 @@ def probe_duration(path: Path) -> int:
|
|||||||
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
|
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
|
||||||
try:
|
try:
|
||||||
out = subprocess.run(
|
out = subprocess.run(
|
||||||
["ffprobe", "-v", "error", "-show_entries", "format=duration",
|
[
|
||||||
"-of", "default=nw=1:nk=1", str(path)],
|
"ffprobe",
|
||||||
capture_output=True, text=True, timeout=30, check=True,
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-show_entries",
|
||||||
|
"format=duration",
|
||||||
|
"-of",
|
||||||
|
"default=nw=1:nk=1",
|
||||||
|
str(path),
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
check=True,
|
||||||
).stdout.strip()
|
).stdout.strip()
|
||||||
return int(float(out)) if out else 0
|
return int(float(out)) if out else 0
|
||||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool) -> tuple[bool, bool, bool]:
|
def browser_transcode_flags(
|
||||||
|
vcodec: str | None, acodec: str | None, is_mp4: bool
|
||||||
|
) -> tuple[bool, bool, bool]:
|
||||||
"""Decide what (if anything) makes a file universally browser-playable. Returns
|
"""Decide what (if anything) makes a file universally browser-playable. Returns
|
||||||
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
|
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
|
||||||
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
|
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
|
||||||
@@ -443,16 +530,34 @@ def browser_transcode_flags(vcodec: str | None, acodec: str | None, is_mp4: bool
|
|||||||
return (rv or ra or not is_mp4), rv, ra
|
return (rv or ra or not is_mp4), rv, ra
|
||||||
|
|
||||||
|
|
||||||
def build_browser_safe_cmd(src: Path, dest: Path, reencode_video: bool, reencode_audio: bool) -> list[str]:
|
def build_browser_safe_cmd(
|
||||||
|
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool, threads: int | None = None
|
||||||
|
) -> list[str]:
|
||||||
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
||||||
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
||||||
pipe:1` streams machine-readable progress for run_ffmpeg."""
|
pipe:1` streams machine-readable progress for run_ffmpeg. `threads` caps the encoder so a
|
||||||
|
re-encode can't claim every core on the shared box (see the worker-child de-prioritization)."""
|
||||||
cmd = [
|
cmd = [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-progress", "pipe:1", "-nostats", "-i", str(src),
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-progress",
|
||||||
|
"pipe:1",
|
||||||
|
"-nostats",
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
]
|
]
|
||||||
cmd += ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"] if reencode_video else ["-c:v", "copy"]
|
cmd += (
|
||||||
|
["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"]
|
||||||
|
if reencode_video
|
||||||
|
else ["-c:v", "copy"]
|
||||||
|
)
|
||||||
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
||||||
|
if threads and threads > 0:
|
||||||
|
cmd += ["-threads", str(threads)]
|
||||||
cmd += ["-movflags", "+faststart", str(dest)]
|
cmd += ["-movflags", "+faststart", str(dest)]
|
||||||
return cmd
|
return cmd
|
||||||
|
|
||||||
@@ -486,25 +591,63 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
|
|||||||
fps = geom["count"] / dur
|
fps = geom["count"] / dur
|
||||||
vf = f"fps={fps:.6f},scale={_SB_TILE_W}:-2,tile={geom['cols']}x{geom['rows']}"
|
vf = f"fps={fps:.6f},scale={_SB_TILE_W}:-2,tile={geom['cols']}x{geom['rows']}"
|
||||||
return [
|
return [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-i", str(src), "-frames:v", "1", "-q:v", "5", "-vf", vf, str(dest),
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-q:v",
|
||||||
|
"5",
|
||||||
|
"-vf",
|
||||||
|
vf,
|
||||||
|
str(dest),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
||||||
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
|
"""Grab one representative frame as a poster JPEG. `-ss` before `-i` seeks fast."""
|
||||||
return [
|
return [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-ss", f"{max(0.0, at_s):.3f}", "-i", str(src),
|
"-y",
|
||||||
"-frames:v", "1", "-q:v", "3", "-vf", "scale=640:-2", str(dest),
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-ss",
|
||||||
|
f"{max(0.0, at_s):.3f}",
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-q:v",
|
||||||
|
"3",
|
||||||
|
"-vf",
|
||||||
|
"scale=640:-2",
|
||||||
|
str(dest),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
|
def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
|
||||||
"""Decode a downloaded thumbnail (webp/png/…) to a single baseline JPEG frame."""
|
"""Decode a downloaded thumbnail (webp/png/…) to a single baseline JPEG frame."""
|
||||||
return [
|
return [
|
||||||
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-nostdin",
|
"ffmpeg",
|
||||||
"-i", str(src), "-frames:v", "1", "-update", "1", str(dest),
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-update",
|
||||||
|
"1",
|
||||||
|
str(dest),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -550,7 +693,10 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
|||||||
return None
|
return None
|
||||||
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
|
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
|
||||||
if err is not None:
|
if err is not None:
|
||||||
log.info("thumbnail->jpg conversion failed (%s); poster falls back to a cut frame", str(err)[:120])
|
log.info(
|
||||||
|
"thumbnail->jpg conversion failed (%s); poster falls back to a cut frame",
|
||||||
|
str(err)[:120],
|
||||||
|
)
|
||||||
return None
|
return None
|
||||||
return dest if dest.exists() else None
|
return dest if dest.exists() else None
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,16 @@ _RULES: tuple[tuple[str, str], ...] = (
|
|||||||
# Some extractors (e.g. Reddit) gate content behind a logged-in account: "Account
|
# Some extractors (e.g. Reddit) gate content behind a logged-in account: "Account
|
||||||
# authentication is required." — an auth wall we deliberately don't cross, not a transient error.
|
# authentication is required." — an auth wall we deliberately don't cross, not a transient error.
|
||||||
("authentication is required", "login_required"),
|
("authentication is required", "login_required"),
|
||||||
|
# Closed-group / private-post auth walls (Facebook groups, Instagram private, …). yt-dlp phrases
|
||||||
|
# these many ways; the common thread is "you must be signed in" or its own suggestion to supply
|
||||||
|
# cookies. Treating the cookie suggestion as an auth signal is what makes a login-walled link read
|
||||||
|
# honestly (and stay non-retryable) when the acting user has no usable cookie for that host.
|
||||||
|
("log in to", "login_required"),
|
||||||
|
("login required", "login_required"),
|
||||||
|
("you must be logged in", "login_required"),
|
||||||
|
("only available for registered users", "login_required"),
|
||||||
|
("use --cookies", "login_required"),
|
||||||
|
("--cookies-from-browser", "login_required"),
|
||||||
# Geo-restricted.
|
# Geo-restricted.
|
||||||
("in your country", "geo_blocked"),
|
("in your country", "geo_blocked"),
|
||||||
("available in your location", "geo_blocked"),
|
("available in your location", "geo_blocked"),
|
||||||
@@ -42,6 +52,10 @@ _RULES: tuple[tuple[str, str], ...] = (
|
|||||||
("account associated with this video has been terminated", "unavailable"),
|
("account associated with this video has been terminated", "unavailable"),
|
||||||
("this video is not available", "unavailable"),
|
("this video is not available", "unavailable"),
|
||||||
("this video has been removed", "unavailable"),
|
("this video has been removed", "unavailable"),
|
||||||
|
# yt-dlp doesn't support this site at all ("Unsupported URL: …") — distinct from an auth wall on a
|
||||||
|
# supported site. Non-retryable: no cookie or retry can make an unsupported host downloadable.
|
||||||
|
("unsupported url", "unsupported_host"),
|
||||||
|
("no video could be found in this url", "unsupported_host"),
|
||||||
# DRM.
|
# DRM.
|
||||||
("drm protected", "drm"),
|
("drm protected", "drm"),
|
||||||
("drm-protected", "drm"),
|
("drm-protected", "drm"),
|
||||||
|
|||||||
@@ -5,14 +5,18 @@ of truth for what file gets produced. `format_sig` hashes exactly the fields tha
|
|||||||
OUTPUT BYTES, so two users picking equivalent settings share one cached MediaAsset.
|
OUTPUT BYTES, so two users picking equivalent settings share one cached MediaAsset.
|
||||||
`build_ydl_opts` turns a spec into a yt-dlp options dict (format selector + postprocessors).
|
`build_ydl_opts` turns a spec into a yt-dlp options dict (format selector + postprocessors).
|
||||||
|
|
||||||
spec shape (all keys optional; defaults applied by `normalize`):
|
spec shape (all keys optional; defaults + browser-playability clamp applied by `normalize`):
|
||||||
mode "av" (audio+video) | "v" (video-only) | "a" (audio-only)
|
mode "av" (audio+video) | "v" (video-only) | "a" (audio-only)
|
||||||
max_height int | null (null = best)
|
max_height int | null (null = best)
|
||||||
container "mp4" | "mkv" | "webm" | null (merge/remux target for av/v)
|
container always coerced to "mp4" (the only universally browser-playable container)
|
||||||
audio_format "m4a" | "mp3" | "opus" | null (extract target for mode "a")
|
audio_format "m4a" | "mp3" (mode "a" extract target; anything else — e.g. opus — → "m4a")
|
||||||
vcodec "h264" | "vp9" | "av1" | null (preference, best-effort)
|
vcodec always coerced to null → H.264 selected (see `normalize`); no opt-out
|
||||||
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
|
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
|
||||||
|
|
||||||
|
`normalize` clamps container/vcodec/audio_format so EVERY download plays in-browser (E7); the
|
||||||
|
`container`/`vcodec` keys survive only so an old stored profile deserializes without error.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
|
||||||
@@ -49,10 +53,12 @@ _DEFAULTS = {
|
|||||||
# prefer H.264+AAC (browser/iOS-compatible) rather than yt-dlp's default VP9/Opus best.
|
# prefer H.264+AAC (browser/iOS-compatible) rather than yt-dlp's default VP9/Opus best.
|
||||||
# v3: mp4 compat now *selects* H.264 (avc1) with a progressive fallback + re-encodes the rare
|
# v3: mp4 compat now *selects* H.264 (avc1) with a progressive fallback + re-encodes the rare
|
||||||
# AV1/VP9-only result to H.264, so cached AV1 assets must re-derive (fixes iOS broken-player).
|
# AV1/VP9-only result to H.264, so cached AV1 assets must re-derive (fixes iOS broken-player).
|
||||||
_SIG_VERSION = 3
|
# v4: EVERY download is clamped to the mp4/H.264/AAC compat profile (no more mkv/webm/VP9/AV1/Opus
|
||||||
|
# opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped
|
||||||
|
# the v3 net — must re-derive through the current browser-safe guarantee.
|
||||||
|
_SIG_VERSION = 4
|
||||||
|
|
||||||
_SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"]
|
_SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"]
|
||||||
_VCODEC_SORT = {"h264": "vcodec:h264", "vp9": "vcodec:vp9", "av1": "vcodec:av01"}
|
|
||||||
|
|
||||||
|
|
||||||
def normalize(spec: dict) -> dict:
|
def normalize(spec: dict) -> dict:
|
||||||
@@ -75,6 +81,18 @@ def normalize(spec: dict) -> dict:
|
|||||||
# Audio-only files can't embed subtitles/thumbnails as video; keep the flags meaningful.
|
# Audio-only files can't embed subtitles/thumbnails as video; keep the flags meaningful.
|
||||||
if s["mode"] == "a":
|
if s["mode"] == "a":
|
||||||
s["embed_subs"] = False
|
s["embed_subs"] = False
|
||||||
|
# Browser-playability guarantee (E7): the Download module exists to download → share → PLAY
|
||||||
|
# in-browser, so every output must play in EVERY browser, iOS/Safari/WebKit included. The only
|
||||||
|
# universally-safe combo is mp4 + H.264 + AAC/MP3, so we CLAMP the spec to it HERE — the single
|
||||||
|
# point every path (picker, stored profile, raw API) flows through — instead of only hiding
|
||||||
|
# options in the UI. Non-mp4 containers (mkv doesn't play at all; webm/VP9/AV1 fail iOS) and any
|
||||||
|
# explicit vcodec are coerced to the compat profile; Opus audio (Safari-flaky as a bare file)
|
||||||
|
# falls back to AAC/m4a. A source that exposes no H.264 is still caught by the worker's
|
||||||
|
# browser-safe re-encode (`_ensure_browser_playable`), so playback is guaranteed end-to-end.
|
||||||
|
s["container"] = "mp4"
|
||||||
|
s["vcodec"] = None
|
||||||
|
if s["audio_format"] not in ("m4a", "mp3"):
|
||||||
|
s["audio_format"] = "m4a"
|
||||||
return s
|
return s
|
||||||
|
|
||||||
|
|
||||||
@@ -97,12 +115,17 @@ def build_ydl_opts(
|
|||||||
outtmpl: str,
|
outtmpl: str,
|
||||||
progress_hook,
|
progress_hook,
|
||||||
extra: dict | None = None,
|
extra: dict | None = None,
|
||||||
|
cookiefile: str | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
|
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
|
||||||
|
|
||||||
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
|
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
|
||||||
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
||||||
generic options below. Everything here is source-agnostic."""
|
generic options below. Everything here is source-agnostic.
|
||||||
|
|
||||||
|
`cookiefile` is the acting user's per-host cookies.txt path (S6), attached ONLY by the worker
|
||||||
|
after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra`
|
||||||
|
merge so a source adapter can never clobber it."""
|
||||||
s = normalize(spec)
|
s = normalize(spec)
|
||||||
h = s["max_height"]
|
h = s["max_height"]
|
||||||
opts: dict = {
|
opts: dict = {
|
||||||
@@ -131,6 +154,11 @@ def build_ydl_opts(
|
|||||||
if extra:
|
if extra:
|
||||||
opts.update(extra)
|
opts.update(extra)
|
||||||
|
|
||||||
|
# The acting user's per-host cookie (S6). AFTER the adapter merge on purpose: an adapter must not
|
||||||
|
# be able to override the authenticated session the worker resolved for this specific user.
|
||||||
|
if cookiefile:
|
||||||
|
opts["cookiefile"] = cookiefile
|
||||||
|
|
||||||
if s["mode"] == "a":
|
if s["mode"] == "a":
|
||||||
opts["format"] = "bestaudio/best"
|
opts["format"] = "bestaudio/best"
|
||||||
opts["postprocessors"].append(
|
opts["postprocessors"].append(
|
||||||
@@ -142,51 +170,33 @@ def build_ydl_opts(
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
hcap = f"[height<=?{h}]" if h else ""
|
hcap = f"[height<=?{h}]" if h else ""
|
||||||
# Compatibility-first for mp4 (the universal container): produce H.264 video + AAC audio
|
# Every video download is the mp4/H.264 compat profile (normalize() clamps container→mp4,
|
||||||
# so shared/downloaded files play in EVERY browser, including iOS/Safari. WebKit (which
|
# vcodec→None), so shared/downloaded files play in EVERY browser, iOS/Safari included. WebKit
|
||||||
# every iOS browser, Brave included, is forced to use) decodes only H.264/H.265 + AAC
|
# (which every iOS browser, Brave included, is forced to use) decodes only H.264/H.265 + AAC
|
||||||
# inside mp4 — a VP9/AV1-in-mp4 file shows a broken player there, even though Chromium
|
# inside mp4 — a VP9/AV1-in-mp4 file shows a broken player there — so H.264 is selected at the
|
||||||
# plays it fine. A custom profile that explicitly sets `vcodec` opts out.
|
# SELECTION level, not merely preferred: a sort key can't conjure an H.264 stream a source
|
||||||
compat = s["container"] == "mp4" and s["vcodec"] is None
|
# doesn't expose as video-only. A source that ships only non-H.264 is caught after download by
|
||||||
|
# the worker's browser-safe re-encode (`_ensure_browser_playable`).
|
||||||
if s["mode"] == "v":
|
if s["mode"] == "v":
|
||||||
if compat:
|
# Video-only: keep it audio-free. Prefer an H.264 video-only stream, then any video-only,
|
||||||
# Video-only: keep it audio-free. Prefer an H.264 *video-only* stream, then any
|
# and only fall to a progressive (audio-bearing) `best` as a last resort.
|
||||||
# video-only, and only fall to a progressive (audio-bearing) `best` as a last
|
opts["format"] = (
|
||||||
# resort — same audio-free-when-possible guarantee as the old `bestvideo/best`.
|
f"bestvideo[vcodec^=avc1]{hcap}/bestvideo{hcap}/best[vcodec^=avc1]{hcap}/best{hcap}"
|
||||||
opts["format"] = (
|
)
|
||||||
f"bestvideo[vcodec^=avc1]{hcap}/bestvideo{hcap}"
|
|
||||||
f"/best[vcodec^=avc1]{hcap}/best{hcap}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
opts["format"] = f"bestvideo{hcap}/best{hcap}"
|
|
||||||
else: # av
|
else: # av
|
||||||
if compat:
|
# avc1 video-only + m4a audio, then the avc1 progressive, then anything (re-encoded).
|
||||||
# Prefer H.264 at the SELECTION level, not just via format_sort: a sort key is only
|
opts["format"] = (
|
||||||
# a *preference* and can't conjure an H.264 stream a source doesn't expose as
|
f"bestvideo[vcodec^=avc1]{hcap}+bestaudio[acodec^=mp4a]"
|
||||||
# video-only. A non-YouTube source (e.g. Facebook) may ship AV1 as the best
|
f"/best[vcodec^=avc1]{hcap}"
|
||||||
# video-only stream and H.264 only as a *progressive* (a+v) format — which
|
f"/bestvideo{hcap}+bestaudio/best{hcap}"
|
||||||
# `bestvideo` skips — so `bestvideo+bestaudio` picked AV1 and iOS broke. This chain
|
)
|
||||||
# tries avc1 video-only, then the avc1 progressive, then anything.
|
opts["merge_output_format"] = "mp4"
|
||||||
opts["format"] = (
|
# Codec preference tiebreaker (ranks H.264/AAC among the selected candidates); the selector
|
||||||
f"bestvideo[vcodec^=avc1]{hcap}+bestaudio[acodec^=mp4a]"
|
# above is the hard guarantee, this is belt-and-braces.
|
||||||
f"/best[vcodec^=avc1]{hcap}"
|
sort = ["vcodec:h264", "acodec:aac"]
|
||||||
f"/bestvideo{hcap}+bestaudio/best{hcap}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
opts["format"] = f"bestvideo{hcap}+bestaudio/best{hcap}"
|
|
||||||
if s["container"]:
|
|
||||||
opts["merge_output_format"] = s["container"]
|
|
||||||
sort = []
|
|
||||||
# Keep the codec preference as a tiebreaker (ranks H.264/AAC above resolution among the
|
|
||||||
# selected candidates); the selector above is the hard guarantee, this is belt-and-braces.
|
|
||||||
if compat:
|
|
||||||
sort += ["vcodec:h264", "acodec:aac"]
|
|
||||||
if h:
|
if h:
|
||||||
sort.append(f"res:{h}")
|
sort.append(f"res:{h}")
|
||||||
if s["vcodec"] in _VCODEC_SORT:
|
opts["format_sort"] = sort
|
||||||
sort.append(_VCODEC_SORT[s["vcodec"]])
|
|
||||||
if sort:
|
|
||||||
opts["format_sort"] = sort
|
|
||||||
|
|
||||||
# SponsorBlock first (marks segments), then strip them from the media + chapters.
|
# SponsorBlock first (marks segments), then strip them from the media + chapters.
|
||||||
if s["sponsorblock"]:
|
if s["sponsorblock"]:
|
||||||
@@ -215,8 +225,6 @@ def build_ydl_opts(
|
|||||||
# jpg/png thumbnail directly and converts anything else to PNG (not the crashing mjpeg2jpeg-to-jpg
|
# jpg/png thumbnail directly and converts anything else to PNG (not the crashing mjpeg2jpeg-to-jpg
|
||||||
# path), so it never needed the convertor and does not reintroduce the crash.
|
# path), so it never needed the convertor and does not reintroduce the crash.
|
||||||
if s["embed_thumbnail"]:
|
if s["embed_thumbnail"]:
|
||||||
opts["postprocessors"].append(
|
opts["postprocessors"].append({"key": "EmbedThumbnail", "already_have_thumbnail": True})
|
||||||
{"key": "EmbedThumbnail", "already_have_thumbnail": True}
|
|
||||||
)
|
|
||||||
|
|
||||||
return opts
|
return opts
|
||||||
|
|||||||
+73
-35
@@ -15,6 +15,7 @@ Five passes each run:
|
|||||||
|
|
||||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
@@ -40,9 +41,7 @@ _ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight do
|
|||||||
def _holders(db: Session, asset_id: int) -> list[int]:
|
def _holders(db: Session, asset_id: int) -> list[int]:
|
||||||
return list(
|
return list(
|
||||||
db.execute(
|
db.execute(
|
||||||
select(DownloadJob.user_id)
|
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
|
||||||
.where(DownloadJob.asset_id == asset_id)
|
|
||||||
.distinct()
|
|
||||||
).scalars()
|
).scalars()
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -87,11 +86,25 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
|||||||
|
|
||||||
cutoff = time.time() - _ORPHAN_MIN_AGE_S
|
cutoff = time.time() - _ORPHAN_MIN_AGE_S
|
||||||
removed = 0
|
removed = 0
|
||||||
|
# Throttle the whole-tree walk: this runs in the API process, and an unpaced rglob+stat over a
|
||||||
|
# large download root monopolizes disk IO long enough to make web requests + playback stutter.
|
||||||
|
# Yield the IO channel after every `batch` files touched so foreground work interleaves.
|
||||||
|
pause_s = max(0, settings.download_gc_sweep_pause_ms) / 1000.0
|
||||||
|
batch = max(1, settings.download_gc_sweep_batch)
|
||||||
|
touched = 0
|
||||||
|
|
||||||
|
def _throttle() -> None:
|
||||||
|
nonlocal touched
|
||||||
|
touched += 1
|
||||||
|
if pause_s and touched % batch == 0:
|
||||||
|
time.sleep(pause_s)
|
||||||
|
|
||||||
for entry in root.iterdir():
|
for entry in root.iterdir():
|
||||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
||||||
continue
|
continue
|
||||||
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
||||||
for f in files:
|
for f in files:
|
||||||
|
_throttle()
|
||||||
try:
|
try:
|
||||||
if f.resolve() in keep or f.stat().st_mtime > cutoff:
|
if f.resolve() in keep or f.stat().st_mtime > cutoff:
|
||||||
continue
|
continue
|
||||||
@@ -101,7 +114,9 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
|||||||
pass
|
pass
|
||||||
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
|
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
|
||||||
for d in sorted(
|
for d in sorted(
|
||||||
(p for p in entry.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True
|
(p for p in entry.rglob("*") if p.is_dir()),
|
||||||
|
key=lambda p: len(p.parts),
|
||||||
|
reverse=True,
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
d.rmdir()
|
d.rmdir()
|
||||||
@@ -121,15 +136,19 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
|
|
||||||
# 1. Pre-expiry warnings.
|
# 1. Pre-expiry warnings.
|
||||||
warn_before = now + timedelta(days=grace_days)
|
warn_before = now + timedelta(days=grace_days)
|
||||||
soon = db.execute(
|
soon = (
|
||||||
select(MediaAsset).where(
|
db.execute(
|
||||||
MediaAsset.status == "ready",
|
select(MediaAsset).where(
|
||||||
MediaAsset.gc_notified.is_(False),
|
MediaAsset.status == "ready",
|
||||||
MediaAsset.expires_at.is_not(None),
|
MediaAsset.gc_notified.is_(False),
|
||||||
MediaAsset.expires_at > now,
|
MediaAsset.expires_at.is_not(None),
|
||||||
MediaAsset.expires_at <= warn_before,
|
MediaAsset.expires_at > now,
|
||||||
|
MediaAsset.expires_at <= warn_before,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
).scalars().all()
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for asset in soon:
|
for asset in soon:
|
||||||
_notify_holders(db, asset, "download_expiring")
|
_notify_holders(db, asset, "download_expiring")
|
||||||
asset.gc_notified = True
|
asset.gc_notified = True
|
||||||
@@ -138,13 +157,17 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
||||||
dead = db.execute(
|
dead = (
|
||||||
select(MediaAsset).where(
|
db.execute(
|
||||||
MediaAsset.status.in_(("ready", "error")),
|
select(MediaAsset).where(
|
||||||
MediaAsset.expires_at.is_not(None),
|
MediaAsset.status.in_(("ready", "error")),
|
||||||
MediaAsset.expires_at <= now,
|
MediaAsset.expires_at.is_not(None),
|
||||||
|
MediaAsset.expires_at <= now,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
).scalars().all()
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for asset in dead:
|
for asset in dead:
|
||||||
_delete_asset(db, asset, "download_expired")
|
_delete_asset(db, asset, "download_expired")
|
||||||
expired += 1
|
expired += 1
|
||||||
@@ -154,17 +177,24 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
# 3. LRU eviction over the total cache cap.
|
# 3. LRU eviction over the total cache cap.
|
||||||
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
||||||
if cap > 0:
|
if cap > 0:
|
||||||
total = db.execute(
|
total = (
|
||||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
db.execute(
|
||||||
MediaAsset.status == "ready"
|
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||||
)
|
MediaAsset.status == "ready"
|
||||||
).scalar() or 0
|
)
|
||||||
|
).scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
if total > cap:
|
if total > cap:
|
||||||
candidates = db.execute(
|
candidates = (
|
||||||
select(MediaAsset)
|
db.execute(
|
||||||
.where(MediaAsset.status == "ready")
|
select(MediaAsset)
|
||||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
.where(MediaAsset.status == "ready")
|
||||||
).scalars().all()
|
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for asset in candidates:
|
for asset in candidates:
|
||||||
if total <= cap:
|
if total <= cap:
|
||||||
break
|
break
|
||||||
@@ -176,13 +206,17 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
||||||
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
||||||
# holder is gone; they carry no expires_at, so passes 1-3 never touch them.
|
# holder is gone; they carry no expires_at, so passes 1-3 never touch them.
|
||||||
orphans = db.execute(
|
orphans = (
|
||||||
select(MediaAsset).where(
|
db.execute(
|
||||||
MediaAsset.status == "error",
|
select(MediaAsset).where(
|
||||||
MediaAsset.ref_count == 0,
|
MediaAsset.status == "error",
|
||||||
MediaAsset.rel_path.is_(None),
|
MediaAsset.ref_count == 0,
|
||||||
|
MediaAsset.rel_path.is_(None),
|
||||||
|
)
|
||||||
)
|
)
|
||||||
).scalars().all()
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
for asset in orphans:
|
for asset in orphans:
|
||||||
db.delete(asset) # no file to unlink; FK SET NULL clears any stray job.asset_id
|
db.delete(asset) # no file to unlink; FK SET NULL clears any stray job.asset_id
|
||||||
reaped += 1
|
reaped += 1
|
||||||
@@ -194,7 +228,11 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
swept = _sweep_orphan_files(db, settings.download_root)
|
swept = _sweep_orphan_files(db, settings.download_root)
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped, "swept": swept
|
"warned": warned,
|
||||||
|
"expired": expired,
|
||||||
|
"evicted": evicted,
|
||||||
|
"reaped": reaped,
|
||||||
|
"swept": swept,
|
||||||
}
|
}
|
||||||
log.info("download_gc %s", result)
|
log.info("download_gc %s", result)
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""Short-lived-session DB helpers + staging paths shared by the worker parent and the child pipeline."""
|
"""Short-lived-session DB helpers + staging paths shared by the worker parent and the child pipeline."""
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
@@ -10,6 +11,7 @@ from app.models import DownloadJob, MediaAsset
|
|||||||
|
|
||||||
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
|
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
|
||||||
|
|
||||||
|
|
||||||
def _set_job(job_id: int, **fields) -> None:
|
def _set_job(job_id: int, **fields) -> None:
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
job = db.get(DownloadJob, job_id)
|
job = db.get(DownloadJob, job_id)
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ require a short-lived signed **grant** on the file request — because a `<video
|
|||||||
header, the password is exchanged once at `/unlock` for an HMAC grant appended to the file URL,
|
header, the password is exchanged once at `/unlock` for an HMAC grant appended to the file URL,
|
||||||
so the raw password never rides in the media URL (or server logs).
|
so the raw password never rides in the media URL (or server logs).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import secrets
|
import secrets
|
||||||
@@ -27,10 +28,9 @@ def is_expired(link: DownloadLink) -> bool:
|
|||||||
|
|
||||||
# --- signed grants (password-protected file access) ----------------------------------------
|
# --- signed grants (password-protected file access) ----------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def _sign(msg: str) -> str:
|
def _sign(msg: str) -> str:
|
||||||
return hmac.new(
|
return hmac.new(settings.secret_key.encode(), msg.encode(), hashlib.sha256).hexdigest()[:32]
|
||||||
settings.secret_key.encode(), msg.encode(), hashlib.sha256
|
|
||||||
).hexdigest()[:32]
|
|
||||||
|
|
||||||
|
|
||||||
def make_grant(token: str, password_version: int, now: float | None = None) -> str:
|
def make_grant(token: str, password_version: int, now: float | None = None) -> str:
|
||||||
@@ -58,6 +58,7 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
|
|||||||
|
|
||||||
# --- serialization -------------------------------------------------------------------------
|
# --- serialization -------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
||||||
"""What the file owner sees when managing their links."""
|
"""What the file owner sees when managing their links."""
|
||||||
return {
|
return {
|
||||||
@@ -72,14 +73,20 @@ def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def public_meta(link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None) -> dict:
|
def public_meta(
|
||||||
|
link: DownloadLink, job, asset, file_url: str, poster_url: str | None = None
|
||||||
|
) -> dict:
|
||||||
"""Metadata for the public watch page (only ever returned once access is authorized).
|
"""Metadata for the public watch page (only ever returned once access is authorized).
|
||||||
|
|
||||||
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
|
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
|
||||||
auto-extracted values, and surfaces the source + any extra reference links as clickable URLs."""
|
auto-extracted values, and surfaces the source + any extra reference links as clickable URLs."""
|
||||||
from app.downloads import service
|
from app.downloads import mediakind, service
|
||||||
|
|
||||||
source_url = service.reference_url(job, asset)
|
source_url = service.reference_url(job, asset)
|
||||||
|
# Audio-only assets (mp3 rips, extracted audio) can't play inside a <video> element on WebKit
|
||||||
|
# (all iOS browsers), so the watch page renders a native <audio> instead. A generic uploaded
|
||||||
|
# file (media_kind='file') has no player at all — the watch page shows file info + a download.
|
||||||
|
# Both branches read the stored media_kind (the single source), not a re-derived codec check.
|
||||||
return {
|
return {
|
||||||
"title": (job and job.display_name) or asset.title,
|
"title": (job and job.display_name) or asset.title,
|
||||||
"uploader": (job and job.display_uploader) or asset.uploader,
|
"uploader": (job and job.display_uploader) or asset.uploader,
|
||||||
@@ -89,6 +96,10 @@ def public_meta(link: DownloadLink, job, asset, file_url: str, poster_url: str |
|
|||||||
"duration_s": asset.duration_s,
|
"duration_s": asset.duration_s,
|
||||||
"width": asset.width,
|
"width": asset.width,
|
||||||
"height": asset.height,
|
"height": asset.height,
|
||||||
|
"audio": asset.media_kind == mediakind.AUDIO,
|
||||||
|
"media_kind": asset.media_kind,
|
||||||
|
"size_bytes": asset.size_bytes,
|
||||||
|
"container": asset.container,
|
||||||
"allow_download": link.allow_download,
|
"allow_download": link.allow_download,
|
||||||
"file_url": file_url,
|
"file_url": file_url,
|
||||||
"poster_url": poster_url,
|
"poster_url": poster_url,
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
"""The media-kind discriminator for a library asset — the single home of one invariant.
|
||||||
|
|
||||||
|
`MediaAsset.media_kind` is `"video" | "audio" | "file"`. It decides which library actions apply:
|
||||||
|
media (video/audio) gets play/edit/share; a generic uploaded file gets download + share only, and
|
||||||
|
is never rendered inline (see the force-download serving in the file/watch routes). Every finalize
|
||||||
|
path computes it HERE — the download worker + the upload ingest via `derive`, the editor by
|
||||||
|
carrying the source's kind over — so the rule lives in code, not in a comment that can drift (an
|
||||||
|
earlier ad-hoc `vcodec is None and acodec is not None` audio check lived only in links.py).
|
||||||
|
"""
|
||||||
|
|
||||||
|
VIDEO = "video"
|
||||||
|
AUDIO = "audio"
|
||||||
|
FILE = "file"
|
||||||
|
|
||||||
|
|
||||||
|
def derive(vcodec: str | None, acodec: str | None) -> str:
|
||||||
|
"""Media kind from the probed codecs: a video stream -> video, audio-only -> audio, neither
|
||||||
|
(a non-media upload: pdf/zip/docx/…) -> file."""
|
||||||
|
if vcodec:
|
||||||
|
return VIDEO
|
||||||
|
if acodec:
|
||||||
|
return AUDIO
|
||||||
|
return FILE
|
||||||
@@ -6,6 +6,7 @@ This injects per-video OG/Twitter tags into the served HTML so a shared link unf
|
|||||||
video's title, channel and thumbnail. Real browsers ignore the extra tags and hydrate the SPA as
|
video's title, channel and thumbnail. Real browsers ignore the extra tags and hydrate the SPA as
|
||||||
usual. Password-protected / expired / invalid links fall back to the generic card (no metadata leak).
|
usual. Password-protected / expired / invalid links fall back to the generic card (no metadata leak).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import html
|
import html
|
||||||
import re
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -71,9 +72,7 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
|
|||||||
|
|
||||||
def _watch_tags(token: str, db: Session) -> str | None:
|
def _watch_tags(token: str, db: Session) -> str | None:
|
||||||
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
|
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
|
||||||
link = db.execute(
|
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
|
||||||
select(DownloadLink).where(DownloadLink.token == token)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if link is None or linksmod.is_expired(link):
|
if link is None or linksmod.is_expired(link):
|
||||||
return None
|
return None
|
||||||
url = f"{settings.app_base}/watch/{token}"
|
url = f"{settings.app_base}/watch/{token}"
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
@@ -7,13 +8,14 @@ from datetime import date, datetime, timedelta, timezone
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
from app import sysconfig
|
from app import sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import errors as dlerrors
|
from app.downloads import errors as dlerrors
|
||||||
from app.downloads import adapters, formats, service, storage
|
from app.downloads import adapters, credentials, formats, mediakind, service, storage
|
||||||
from app.downloads.jobdb import (
|
from app.downloads.jobdb import (
|
||||||
_edit_staging_dir,
|
_edit_staging_dir,
|
||||||
_job_status,
|
_job_status,
|
||||||
@@ -57,9 +59,7 @@ _YTDLP_TEMP_RE = re.compile(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _notify_download_done(
|
def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *, kind: str) -> None:
|
||||||
db, user_id: int | None, title: str | None, job_id, *, kind: str
|
|
||||||
) -> None:
|
|
||||||
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
||||||
silently while scheduler jobs and playlist-push already notified (R11 feedback symmetry). A
|
silently while scheduler jobs and playlist-push already notified (R11 feedback symmetry). A
|
||||||
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
||||||
@@ -113,7 +113,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
|||||||
|
|
||||||
def _make_progress_hook(job_id: int, asset_id: int):
|
def _make_progress_hook(job_id: int, asset_id: int):
|
||||||
state = {"meta_done": False}
|
state = {"meta_done": False}
|
||||||
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
|
cancelled = _cancel_poll(job_id) # same policy as the ffmpeg path
|
||||||
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S)
|
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S)
|
||||||
|
|
||||||
def hook(d: dict) -> None:
|
def hook(d: dict) -> None:
|
||||||
@@ -247,7 +247,9 @@ def _make_pp_hook(job_id: int):
|
|||||||
|
|
||||||
def hook(d: dict) -> None:
|
def hook(d: dict) -> None:
|
||||||
if d.get("status") in ("started", "processing"):
|
if d.get("status") in ("started", "processing"):
|
||||||
_set_job(job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None)
|
_set_job(
|
||||||
|
job_id, phase=_pp_phase(d.get("postprocessor") or ""), speed_bps=None, eta_s=None
|
||||||
|
)
|
||||||
|
|
||||||
return hook
|
return hook
|
||||||
|
|
||||||
@@ -269,7 +271,9 @@ def _produced_file(info: dict, staging: Path) -> Path:
|
|||||||
back to the largest file left in staging that is neither a thumbnail nor one of yt-dlp's own
|
back to the largest file left in staging that is neither a thumbnail nor one of yt-dlp's own
|
||||||
working files (see `_YTDLP_TEMP_RE`)."""
|
working files (see `_YTDLP_TEMP_RE`)."""
|
||||||
req = (info.get("requested_downloads") or [{}])[0]
|
req = (info.get("requested_downloads") or [{}])[0]
|
||||||
fp = req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
|
fp = (
|
||||||
|
req.get("filepath") or req.get("_filename") or info.get("filepath") or info.get("_filename")
|
||||||
|
)
|
||||||
if fp:
|
if fp:
|
||||||
produced = Path(str(fp))
|
produced = Path(str(fp))
|
||||||
if produced.is_file():
|
if produced.is_file():
|
||||||
@@ -301,17 +305,23 @@ def _media_info(info: dict) -> dict:
|
|||||||
return {**info, **chosen}
|
return {**info, **chosen}
|
||||||
|
|
||||||
|
|
||||||
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter, cookiefile=None):
|
||||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||||
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
|
error); other sources make a single attempt. Returns the info dict of the successful attempt.
|
||||||
|
|
||||||
|
`cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by
|
||||||
|
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated."""
|
||||||
client_sets = adapter.client_sets()
|
client_sets = adapter.client_sets()
|
||||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for i, clients in enumerate(client_sets):
|
for i, clients in enumerate(client_sets):
|
||||||
opts = formats.build_ydl_opts(
|
opts = formats.build_ydl_opts(
|
||||||
spec, outtmpl, _make_progress_hook(job_id, asset_id),
|
spec,
|
||||||
|
outtmpl,
|
||||||
|
_make_progress_hook(job_id, asset_id),
|
||||||
extra=adapter.ydl_opts_extra(clients),
|
extra=adapter.ydl_opts_extra(clients),
|
||||||
|
cookiefile=cookiefile,
|
||||||
)
|
)
|
||||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||||
@@ -324,7 +334,12 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
|||||||
except yt_dlp.utils.DownloadError as exc:
|
except yt_dlp.utils.DownloadError as exc:
|
||||||
last_exc = exc
|
last_exc = exc
|
||||||
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||||
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
|
log.info(
|
||||||
|
"job %s: clients %s failed (%s); retrying with fallback",
|
||||||
|
job_id,
|
||||||
|
clients,
|
||||||
|
str(exc)[:70],
|
||||||
|
)
|
||||||
for p in list(staging.iterdir()): # drop partials before the retry
|
for p in list(staging.iterdir()): # drop partials before the retry
|
||||||
try:
|
try:
|
||||||
p.unlink()
|
p.unlink()
|
||||||
@@ -363,13 +378,17 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
|||||||
|
|
||||||
dest = produced.with_name(produced.stem + ".browser.mp4")
|
dest = produced.with_name(produced.stem + ".browser.mp4")
|
||||||
action = "re-encode" if (reenc_v or reenc_a) else "remux"
|
action = "re-encode" if (reenc_v or reenc_a) else "remux"
|
||||||
log.info("job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4)
|
log.info(
|
||||||
|
"job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4
|
||||||
|
)
|
||||||
_set_job(job_id, phase="processing", progress=0, speed_bps=None, eta_s=None)
|
_set_job(job_id, phase="processing", progress=0, speed_bps=None, eta_s=None)
|
||||||
# Prefer the info duration; probe the file when the extractor didn't provide one, so a long
|
# Prefer the info duration; probe the file when the extractor didn't provide one, so a long
|
||||||
# re-encode still shows real progress instead of a frozen 0%.
|
# re-encode still shows real progress instead of a frozen 0%.
|
||||||
total_s = int(info["duration"]) if info.get("duration") else editmod.probe_duration(produced)
|
total_s = int(info["duration"]) if info.get("duration") else editmod.probe_duration(produced)
|
||||||
editmod.run_ffmpeg(
|
editmod.run_ffmpeg(
|
||||||
editmod.build_browser_safe_cmd(produced, dest, reenc_v, reenc_a),
|
editmod.build_browser_safe_cmd(
|
||||||
|
produced, dest, reenc_v, reenc_a, threads=editmod.encode_thread_cap()
|
||||||
|
),
|
||||||
total_s=total_s,
|
total_s=total_s,
|
||||||
on_progress=_ffmpeg_progress_writer(job_id, "processing"),
|
on_progress=_ffmpeg_progress_writer(job_id, "processing"),
|
||||||
should_cancel=_cancel_poll(job_id),
|
should_cancel=_cancel_poll(job_id),
|
||||||
@@ -383,8 +402,11 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
|||||||
return dest
|
return dest
|
||||||
|
|
||||||
|
|
||||||
def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict) -> None:
|
def _download(
|
||||||
|
job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict, user_id: int
|
||||||
|
) -> None:
|
||||||
staging = _staging_dir(asset_id)
|
staging = _staging_dir(asset_id)
|
||||||
|
cookiefile: Path | None = None
|
||||||
try:
|
try:
|
||||||
if staging.exists():
|
if staging.exists():
|
||||||
shutil.rmtree(staging, ignore_errors=True)
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
@@ -393,7 +415,27 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
|||||||
_set_job(job_id, phase="downloading", progress=0)
|
_set_job(job_id, phase="downloading", progress=0)
|
||||||
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
||||||
url = service.source_url(source_kind, source_ref)
|
url = service.source_url(source_kind, source_ref)
|
||||||
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
|
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||||
|
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||||
|
with SessionLocal() as cdb:
|
||||||
|
asset_sig = cdb.execute(
|
||||||
|
select(MediaAsset.format_sig).where(MediaAsset.id == asset_id)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if asset_sig is not None:
|
||||||
|
cookiefile = credentials.resolve_cookiefile(
|
||||||
|
cdb, user_id, url, asset_sig, spec, staging
|
||||||
|
)
|
||||||
|
info = _media_info(
|
||||||
|
_extract_with_fallback(
|
||||||
|
job_id,
|
||||||
|
asset_id,
|
||||||
|
url,
|
||||||
|
spec,
|
||||||
|
staging,
|
||||||
|
adapter,
|
||||||
|
cookiefile=str(cookiefile) if cookiefile else None,
|
||||||
|
)
|
||||||
|
)
|
||||||
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
||||||
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
||||||
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||||
@@ -436,8 +478,13 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
|||||||
asset.width = int(info["width"]) if info.get("width") else None
|
asset.width = int(info["width"]) if info.get("width") else None
|
||||||
asset.height = int(info["height"]) if info.get("height") else None
|
asset.height = int(info["height"]) if info.get("height") else None
|
||||||
asset.container = ext
|
asset.container = ext
|
||||||
asset.vcodec = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
asset.vcodec = (
|
||||||
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||||
|
)
|
||||||
|
asset.acodec = (
|
||||||
|
info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||||
|
)
|
||||||
|
asset.media_kind = mediakind.derive(asset.vcodec, asset.acodec)
|
||||||
asset.title = meta.title
|
asset.title = meta.title
|
||||||
asset.uploader = meta.uploader
|
asset.uploader = meta.uploader
|
||||||
asset.uploader_url = adapter.uploader_url(info)
|
asset.uploader_url = adapter.uploader_url(info)
|
||||||
@@ -456,8 +503,13 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
|||||||
job = db.get(DownloadJob, job_id)
|
job = db.get(DownloadJob, job_id)
|
||||||
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
||||||
owner_id = job.user_id if job else None
|
owner_id = job.user_id if job else None
|
||||||
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None, "speed_bps": None,
|
fields = {
|
||||||
"eta_s": None}
|
"status": JobStatus.DONE,
|
||||||
|
"progress": 100,
|
||||||
|
"phase": None,
|
||||||
|
"speed_bps": None,
|
||||||
|
"eta_s": None,
|
||||||
|
}
|
||||||
if job and not job.display_name and meta.title:
|
if job and not job.display_name and meta.title:
|
||||||
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
||||||
fields["display_name"] = meta.title[:255]
|
fields["display_name"] = meta.title[:255]
|
||||||
@@ -484,12 +536,20 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
|||||||
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
|
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
|
||||||
# with `error` (error also holds the asset → would re-open the double-release class).
|
# with `error` (error also holds the asset → would re-open the double-release class).
|
||||||
service.finalize_if_running(
|
service.finalize_if_running(
|
||||||
db, job_id, status=JobStatus.ERROR, error=msg,
|
db,
|
||||||
error_code=dlerrors.classify(msg), phase=None,
|
job_id,
|
||||||
|
status=JobStatus.ERROR,
|
||||||
|
error=msg,
|
||||||
|
error_code=dlerrors.classify(msg),
|
||||||
|
phase=None,
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
log.warning("job %s failed: %s", job_id, msg)
|
log.warning("job %s failed: %s", job_id, msg)
|
||||||
finally:
|
finally:
|
||||||
|
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
|
||||||
|
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
|
||||||
|
if cookiefile is not None:
|
||||||
|
cookiefile.unlink(missing_ok=True)
|
||||||
shutil.rmtree(staging, ignore_errors=True)
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
@@ -501,8 +561,12 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
|||||||
asset.error = msg
|
asset.error = msg
|
||||||
# Atomic fail: only while still running (see the download error path).
|
# Atomic fail: only while still running (see the download error path).
|
||||||
service.finalize_if_running(
|
service.finalize_if_running(
|
||||||
db, job_id, status=JobStatus.ERROR, error=msg,
|
db,
|
||||||
error_code=dlerrors.classify(msg), phase=None,
|
job_id,
|
||||||
|
status=JobStatus.ERROR,
|
||||||
|
error=msg,
|
||||||
|
error_code=dlerrors.classify(msg),
|
||||||
|
phase=None,
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
log.warning("edit job %s failed: %s", job_id, msg)
|
log.warning("edit job %s failed: %s", job_id, msg)
|
||||||
@@ -517,7 +581,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
|||||||
user_id = job.user_id if job else None
|
user_id = job.user_id if job else None
|
||||||
src_ready = src is not None and src.status == "ready" and bool(src.rel_path)
|
src_ready = src is not None and src.status == "ready" and bool(src.rel_path)
|
||||||
src_rel = src.rel_path if src else None
|
src_rel = src.rel_path if src else None
|
||||||
src_w, src_h, src_dur = (src.width, src.height, src.duration_s) if src else (None, None, None)
|
src_w, src_h, src_dur = (
|
||||||
|
(src.width, src.height, src.duration_s) if src else (None, None, None)
|
||||||
|
)
|
||||||
|
|
||||||
if not src_ready or not src_rel:
|
if not src_ready or not src_rel:
|
||||||
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
||||||
@@ -541,7 +607,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
|||||||
|
|
||||||
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
||||||
if edit_spec.get("segments"):
|
if edit_spec.get("segments"):
|
||||||
cmd, prep = editmod.build_concat_plan(src_path, dest_staging, edit_spec, out_ext, staging)
|
cmd, prep = editmod.build_concat_plan(
|
||||||
|
src_path, dest_staging, edit_spec, out_ext, staging
|
||||||
|
)
|
||||||
for p, content in prep.items():
|
for p, content in prep.items():
|
||||||
p.write_text(content, encoding="utf-8")
|
p.write_text(content, encoding="utf-8")
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Limits come from a per-user DownloadQuota row if the admin set one, else the sys
|
|||||||
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
|
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
|
||||||
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
|
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
@@ -121,6 +122,22 @@ def check_enqueue(db: Session, user_id: int) -> None:
|
|||||||
raise QuotaExceeded("max_bytes", lim.max_bytes, used)
|
raise QuotaExceeded("max_bytes", lim.max_bytes, used)
|
||||||
|
|
||||||
|
|
||||||
|
def check_upload(db: Session, user_id: int, add_bytes: int) -> None:
|
||||||
|
"""Like check_enqueue, but for an upload whose size is KNOWN at ingest: also reject when adding
|
||||||
|
this file would push the footprint over max_bytes. A download can't pre-check its size (unknown
|
||||||
|
until the worker finishes) so check_enqueue only compares the current footprint; an upload can,
|
||||||
|
so it's charged against the cap before it lands. Raises QuotaExceeded(max_jobs|max_bytes)."""
|
||||||
|
lim = resolve(db, user_id)
|
||||||
|
if lim.unlimited:
|
||||||
|
return
|
||||||
|
n = active_jobs(db, user_id)
|
||||||
|
if n >= lim.max_jobs:
|
||||||
|
raise QuotaExceeded("max_jobs", lim.max_jobs, n)
|
||||||
|
projected = footprint(db, user_id) + add_bytes
|
||||||
|
if projected > lim.max_bytes:
|
||||||
|
raise QuotaExceeded("max_bytes", lim.max_bytes, projected)
|
||||||
|
|
||||||
|
|
||||||
def usage(db: Session, user_id: int) -> dict:
|
def usage(db: Session, user_id: int) -> dict:
|
||||||
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
|
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
|
||||||
lim = resolve(db, user_id)
|
lim = resolve(db, user_id)
|
||||||
|
|||||||
@@ -10,17 +10,50 @@ Contract: the asset is already claimed ('downloading') by the parent. This proce
|
|||||||
writes the terminal job/asset state itself (done / error, or a cooperative pause via the in-flight
|
writes the terminal job/asset state itself (done / error, or a cooperative pause via the in-flight
|
||||||
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
|
from app.downloads import edit as editmod
|
||||||
from app.models import DownloadJob
|
from app.models import DownloadJob
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.worker.runner")
|
log = logging.getLogger("siftlode.worker.runner")
|
||||||
|
|
||||||
|
|
||||||
|
def _deprioritize() -> None:
|
||||||
|
"""Lower THIS child's CPU + IO priority so its yt-dlp/ffmpeg work — every descendant inherits the
|
||||||
|
handicap — can never starve the co-located API, Postgres, or Plex playback. Run once at startup,
|
||||||
|
before any heavy work. Every step is best-effort: a failed renice must never fail a download."""
|
||||||
|
nice_inc = settings.download_child_nice
|
||||||
|
if nice_inc > 0:
|
||||||
|
try:
|
||||||
|
os.nice(nice_inc) # relative increment; the child starts at the worker's nice (0)
|
||||||
|
except OSError as exc:
|
||||||
|
log.info("could not renice download child: %s", str(exc)[:120])
|
||||||
|
cls = settings.download_child_ionice_class
|
||||||
|
if cls > 0 and shutil.which("ionice"): # util-linux; absent on some minimal images
|
||||||
|
try:
|
||||||
|
subprocess.run(
|
||||||
|
["ionice", "-c", str(cls), "-p", str(os.getpid())],
|
||||||
|
capture_output=True,
|
||||||
|
timeout=5,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.SubprocessError) as exc:
|
||||||
|
log.info("could not ionice download child: %s", str(exc)[:120])
|
||||||
|
# Cap OpenMP-parallel ffmpeg filters to the same budget as the encoder threads, so a filter
|
||||||
|
# graph can't fan out across every core behind ffmpeg's own `-threads` cap.
|
||||||
|
os.environ.setdefault("OMP_NUM_THREADS", str(editmod.encode_thread_cap()))
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
_deprioritize()
|
||||||
if len(sys.argv) != 2:
|
if len(sys.argv) != 2:
|
||||||
print("usage: python -m app.downloads.runner <job_id>", file=sys.stderr)
|
print("usage: python -m app.downloads.runner <job_id>", file=sys.stderr)
|
||||||
raise SystemExit(2)
|
raise SystemExit(2)
|
||||||
@@ -34,6 +67,7 @@ def main() -> None:
|
|||||||
job_kind = job.job_kind
|
job_kind = job.job_kind
|
||||||
spec = dict(job.profile_snapshot or {})
|
spec = dict(job.profile_snapshot or {})
|
||||||
source_kind, source_ref = job.source_kind, job.source_ref
|
source_kind, source_ref = job.source_kind, job.source_ref
|
||||||
|
user_id = job.user_id
|
||||||
if asset_id is None:
|
if asset_id is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -44,7 +78,7 @@ def main() -> None:
|
|||||||
if job_kind == "edit":
|
if job_kind == "edit":
|
||||||
_process_edit(job_id, asset_id)
|
_process_edit(job_id, asset_id)
|
||||||
else:
|
else:
|
||||||
_download(job_id, asset_id, source_kind, source_ref, spec)
|
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ cache hit (asset already ready) — marks the job done instantly with no re-down
|
|||||||
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
|
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
|
||||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import func, select, update
|
||||||
@@ -14,8 +15,9 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import adapters
|
from app.downloads import adapters
|
||||||
|
from app.downloads import credentials
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import formats, quota, storage
|
from app.downloads import formats, mediakind, quota, storage
|
||||||
from app.downloads.status import HOLDING, JobStatus
|
from app.downloads.status import HOLDING, JobStatus
|
||||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||||
|
|
||||||
@@ -154,7 +156,12 @@ def enqueue(
|
|||||||
Raises quota.QuotaExceeded if the user is at their job-count or footprint cap."""
|
Raises quota.QuotaExceeded if the user is at their job-count or footprint cap."""
|
||||||
quota.check_enqueue(db, user_id)
|
quota.check_enqueue(db, user_id)
|
||||||
spec = formats.normalize(spec)
|
spec = formats.normalize(spec)
|
||||||
sig = formats.format_sig(spec)
|
# If this user has a stored cookie for the download's host, salt the cache signature with their
|
||||||
|
# id so the cookie-authed result is a PRIVATE asset — never dedup-shared with, or cache-served
|
||||||
|
# to, another user (the load-bearing privacy fix; the worker attaches the cookie to exactly this
|
||||||
|
# salted identity). A normal download (no matching cookie) gets salt=None and shares as before.
|
||||||
|
salt = credentials.salt_for_enqueue(db, user_id, source_url(source_kind, source_ref))
|
||||||
|
sig = formats.format_sig(spec, salt=salt)
|
||||||
asset = get_or_create_asset(db, source_kind, source_ref, sig)
|
asset = get_or_create_asset(db, source_kind, source_ref, sig)
|
||||||
# Show a real title + thumbnail the moment it's queued (feed downloads are catalog videos).
|
# Show a real title + thumbnail the moment it's queued (feed downloads are catalog videos).
|
||||||
populate_from_catalog(db, asset)
|
populate_from_catalog(db, asset)
|
||||||
@@ -166,7 +173,8 @@ def enqueue(
|
|||||||
source_ref=source_ref,
|
source_ref=source_ref,
|
||||||
profile_id=profile_id,
|
profile_id=profile_id,
|
||||||
profile_snapshot=spec,
|
profile_snapshot=spec,
|
||||||
display_name=display_name or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
display_name=display_name
|
||||||
|
or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
||||||
status="queued",
|
status="queued",
|
||||||
queue_pos=next_queue_pos(db),
|
queue_pos=next_queue_pos(db),
|
||||||
)
|
)
|
||||||
@@ -208,6 +216,10 @@ def enqueue_edit(
|
|||||||
src_asset = db.get(MediaAsset, source_job.asset_id) if source_job.asset_id else None
|
src_asset = db.get(MediaAsset, source_job.asset_id) if source_job.asset_id else None
|
||||||
if src_asset is None or src_asset.status != "ready" or not src_asset.rel_path:
|
if src_asset is None or src_asset.status != "ready" or not src_asset.rel_path:
|
||||||
raise EditError("source_not_ready")
|
raise EditError("source_not_ready")
|
||||||
|
# A generic uploaded file (media_kind='file') has no timeline to trim/crop — the editor path is
|
||||||
|
# media-only. Reject here so an upload's file never reaches the ffmpeg edit worker.
|
||||||
|
if src_asset.media_kind == mediakind.FILE:
|
||||||
|
raise EditError("not_media")
|
||||||
|
|
||||||
spec = editmod.normalize_edit_spec(edit_spec)
|
spec = editmod.normalize_edit_spec(edit_spec)
|
||||||
if not spec:
|
if not spec:
|
||||||
@@ -224,6 +236,9 @@ def enqueue_edit(
|
|||||||
asset.thumbnail_url = src_asset.thumbnail_url
|
asset.thumbnail_url = src_asset.thumbnail_url
|
||||||
asset.upload_date = src_asset.upload_date
|
asset.upload_date = src_asset.upload_date
|
||||||
asset.duration_s = editmod.clip_duration(spec, src_asset.duration_s)
|
asset.duration_s = editmod.clip_duration(spec, src_asset.duration_s)
|
||||||
|
# A clip inherits its source's kind (a video edit stays video, an audio trim stays audio);
|
||||||
|
# the ffmpeg edit worker doesn't re-probe codecs, so set it here where the source is known.
|
||||||
|
asset.media_kind = src_asset.media_kind or mediakind.VIDEO
|
||||||
|
|
||||||
job = DownloadJob(
|
job = DownloadJob(
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ Plus a Kodi/Plex-readable `<basename>.nfo` (episodedetails) and `<basename>.jpg`
|
|||||||
a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode,
|
a Plex library using a metadata agent picks up rich info. Channel = "show", video = episode,
|
||||||
date-based — the community-standard mapping for YouTube-in-Plex.
|
date-based — the community-standard mapping for YouTube-in-Plex.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import unicodedata
|
import unicodedata
|
||||||
@@ -18,6 +19,8 @@ from datetime import date
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from xml.sax.saxutils import escape
|
from xml.sax.saxutils import escape
|
||||||
|
|
||||||
|
from app.downloads import mediakind
|
||||||
|
|
||||||
# Reserve room for the "_id.ext" suffix + parent dirs within the 255-char component limit.
|
# Reserve room for the "_id.ext" suffix + parent dirs within the 255-char component limit.
|
||||||
_MAX_TITLE = 120
|
_MAX_TITLE = 120
|
||||||
# User-facing (Content-Disposition) names can be longer than an on-disk component.
|
# User-facing (Content-Disposition) names can be longer than an on-disk component.
|
||||||
@@ -36,9 +39,27 @@ _DROP_CATEGORIES = {"So", "Sk", "Cc", "Cf", "Cs", "Co", "Cn"}
|
|||||||
# that accented names show in non-UTF-8 tools.
|
# that accented names show in non-UTF-8 tools.
|
||||||
_FOLD_MAP = str.maketrans(
|
_FOLD_MAP = str.maketrans(
|
||||||
{
|
{
|
||||||
"ß": "ss", "ẞ": "SS", "ø": "o", "Ø": "O", "ł": "l", "Ł": "L", "đ": "d", "Đ": "D",
|
"ß": "ss",
|
||||||
"ð": "d", "Ð": "D", "þ": "th", "Þ": "Th", "æ": "ae", "Æ": "AE", "œ": "oe", "Œ": "OE",
|
"ẞ": "SS",
|
||||||
"ı": "i", "İ": "I", "·": " ", "–": "-", "—": "-",
|
"ø": "o",
|
||||||
|
"Ø": "O",
|
||||||
|
"ł": "l",
|
||||||
|
"Ł": "L",
|
||||||
|
"đ": "d",
|
||||||
|
"Đ": "D",
|
||||||
|
"ð": "d",
|
||||||
|
"Ð": "D",
|
||||||
|
"þ": "th",
|
||||||
|
"Þ": "Th",
|
||||||
|
"æ": "ae",
|
||||||
|
"Æ": "AE",
|
||||||
|
"œ": "oe",
|
||||||
|
"Œ": "OE",
|
||||||
|
"ı": "i",
|
||||||
|
"İ": "I",
|
||||||
|
"·": " ",
|
||||||
|
"–": "-",
|
||||||
|
"—": "-",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -128,6 +149,81 @@ def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
|
|||||||
return f".edits/{user_id}/clip_{asset_id}.{ext}"
|
return f".edits/{user_id}/clip_{asset_id}.{ext}"
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_upload_ext(filename: str) -> str:
|
||||||
|
"""The upload's extension reduced to a shell-safe, lowercased, bounded token (``""`` when the
|
||||||
|
upload has no extension). The SINGLE definition feeding both the on-disk suffix and the stored
|
||||||
|
`container` — they must agree (the served Content-Disposition suffix has to match the container),
|
||||||
|
so the sanitize/truncation rule lives in one place, not two drifting copies."""
|
||||||
|
return _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
|
||||||
|
"""On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
|
||||||
|
|
||||||
|
Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of
|
||||||
|
the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster
|
||||||
|
episode sidecars). The name is system-decided and bound to the asset id (`_a{id}` keeps two
|
||||||
|
same-named uploads apart); the original filename is display metadata only. The extension is
|
||||||
|
preserved (shell-safe, bounded) so the served file + its Content-Disposition carry a correct
|
||||||
|
suffix; an extensionless upload stays extensionless."""
|
||||||
|
stem = sanitize(Path(filename).stem, 80)
|
||||||
|
ext = _clean_upload_ext(filename)
|
||||||
|
suffix = f".{ext}" if ext else ""
|
||||||
|
return f".uploads/{user_id}/{stem}_a{asset_id}{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def upload_ext(filename: str) -> str | None:
|
||||||
|
"""The stored `container` for an uploaded file: its shell-safe lowercased extension, or None when
|
||||||
|
the upload has no extension (a bare name). Shares `_clean_upload_ext` with `upload_rel_path` so the
|
||||||
|
stored container and the on-disk suffix can never diverge."""
|
||||||
|
return _clean_upload_ext(filename) or None
|
||||||
|
|
||||||
|
|
||||||
|
# The Content-Type used to serve an inline-playable media asset, keyed by its stored `container`.
|
||||||
|
# DELIBERATELY a strict allow-list of known audio/video containers: the served type is chosen from
|
||||||
|
# the asset's ffprobe-verified media_kind + this table, NEVER guessed from the raw (user-controlled)
|
||||||
|
# upload extension. Without this, an uploaded media/HTML polyglot named "evil.html" — a real MP3/MP4
|
||||||
|
# that ffprobe classifies as media, so it dodges the generic-file force-download — would be served
|
||||||
|
# with a Content-Type Starlette guesses from ".html" (text/html) and execute as stored XSS on the
|
||||||
|
# serving origin. A media asset whose container isn't in this table falls back to a neutral
|
||||||
|
# force-download attachment (see `served_media_type`), so nothing script-executable is ever inline.
|
||||||
|
_MEDIA_MIME: dict[str, str] = {
|
||||||
|
# video containers
|
||||||
|
"mp4": "video/mp4",
|
||||||
|
"m4v": "video/mp4",
|
||||||
|
"mov": "video/quicktime",
|
||||||
|
"webm": "video/webm",
|
||||||
|
"mkv": "video/x-matroska",
|
||||||
|
"ogv": "video/ogg",
|
||||||
|
"avi": "video/x-msvideo",
|
||||||
|
# audio containers
|
||||||
|
"m4a": "audio/mp4",
|
||||||
|
"mp3": "audio/mpeg",
|
||||||
|
"aac": "audio/aac",
|
||||||
|
"ogg": "audio/ogg",
|
||||||
|
"oga": "audio/ogg",
|
||||||
|
"opus": "audio/ogg",
|
||||||
|
"weba": "audio/webm",
|
||||||
|
"wav": "audio/wav",
|
||||||
|
"flac": "audio/flac",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def served_media_type(media_kind: str | None, container: str | None) -> str | None:
|
||||||
|
"""The Content-Type a library file MUST be served with — a real audio/video MIME for a recognised
|
||||||
|
media container, else ``None`` meaning "serve as a neutral force-download attachment".
|
||||||
|
|
||||||
|
Returns None for a generic ``file`` kind AND for any media whose container we don't positively
|
||||||
|
recognise as safe. Callers serve None as ``application/octet-stream`` + ``attachment`` so the
|
||||||
|
served Content-Type is NEVER derived from the (attacker-controlled) file extension — the guard
|
||||||
|
that stops uploaded HTML/SVG or a media/HTML polyglot from executing inline on the serving
|
||||||
|
origin. A ``None`` media_kind (a legacy asset before the discriminator was backfilled) is treated
|
||||||
|
as media so an old download keeps its inline player."""
|
||||||
|
if media_kind == mediakind.FILE:
|
||||||
|
return None
|
||||||
|
return _MEDIA_MIME.get((container or "").lower())
|
||||||
|
|
||||||
|
|
||||||
def download_filename(display_name: str, container: str | None, path: Path) -> str:
|
def download_filename(display_name: str, container: str | None, path: Path) -> str:
|
||||||
"""The Content-Disposition filename for a served download: the cleaned display name carrying a single
|
"""The Content-Disposition filename for a served download: the cleaned display name carrying a single
|
||||||
correct extension — the asset container, else the file's own suffix — without doubling it when the
|
correct extension — the asset container, else the file's own suffix — without doubling it when the
|
||||||
@@ -174,7 +270,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
|
|||||||
f" <premiered>{aired}</premiered>",
|
f" <premiered>{aired}</premiered>",
|
||||||
f" <year>{year}</year>",
|
f" <year>{year}</year>",
|
||||||
f" <plot>{escape(meta.description or '')}</plot>",
|
f" <plot>{escape(meta.description or '')}</plot>",
|
||||||
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
f' <uniqueid type="{escape(meta.uniqueid_type)}" default="true">{escape(meta.video_id)}</uniqueid>',
|
||||||
]
|
]
|
||||||
if meta.duration_s:
|
if meta.duration_s:
|
||||||
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
||||||
@@ -190,9 +286,7 @@ def _sidecar(media: Path, ext: str) -> Path:
|
|||||||
return media.with_suffix(ext)
|
return media.with_suffix(ext)
|
||||||
|
|
||||||
|
|
||||||
def write_sidecars(
|
def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
|
||||||
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
|
|
||||||
) -> bool:
|
|
||||||
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
||||||
media = abs_path(download_root, rel)
|
media = abs_path(download_root, rel)
|
||||||
try:
|
try:
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user