Compare commits
24
Commits
v0.107.3
...
a35eece652
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a35eece652 | ||
|
|
b703234c17 | ||
|
|
dbb4065e7b | ||
|
|
e82f87acb0 | ||
|
|
e9c60f72cd | ||
|
|
3a688fa0f5 | ||
|
|
d66174ed16 | ||
|
|
c85ae42b31 | ||
|
|
cb9a89682a | ||
|
|
b842e4e871 | ||
|
|
5082636f15 | ||
|
|
13b7577d9f | ||
|
|
fad656988a | ||
|
|
3021be5166 | ||
|
|
c288ba147b | ||
|
|
fe6308d844 | ||
|
|
24448e7c6f | ||
|
|
d13483a593 | ||
|
|
3bab2e7dc7 | ||
|
|
1b1e3cdcf2 | ||
|
|
27c2fcbd2e | ||
|
|
911527a548 | ||
|
|
8fd91b7b15 | ||
|
|
45c0ee7d58 |
+5
-3
@@ -68,9 +68,11 @@ SMTP_FROM=
|
||||
|
||||
# --- Scheduler ---
|
||||
# The background scheduler (subscription sync, backfill, enrichment) runs inside the app.
|
||||
# DATABASE_URL is set for you by docker-compose to the bundled `db` service. If you ever run
|
||||
# more than one instance against the same database, keep SCHEDULER_ENABLED=true on exactly one
|
||||
# of them and false on the rest, to avoid double quota use and write races.
|
||||
# DATABASE_URL is set for you by docker-compose to the bundled `db` service. Run ONE app instance
|
||||
# per database: a starting instance closes every run still open as interrupted (it assumes the
|
||||
# previous process is gone), and "is this job already running" is held in the process. A second
|
||||
# instance would cut the first one's runs short and could run the same job twice.
|
||||
# SCHEDULER_ENABLED=false keeps the recurring jobs off; "run now" still works.
|
||||
SCHEDULER_ENABLED=true
|
||||
|
||||
# --- Files library & downloads ---
|
||||
|
||||
@@ -7,6 +7,19 @@ edit the release notes and re-run the generator.
|
||||
|
||||
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
||||
|
||||
## 0.108.0 — 2026-10-18
|
||||
|
||||
### New
|
||||
|
||||
- Activity: one admin page for background runs, admin actions, the server log and quota, with one time range, account and search. A run opens to its outcome and its log lines, an admin action to its request's lines; old audit and log links open their tab.
|
||||
- Scheduler: each job shows its last three runs (a run cut short by a restart is marked) and a button opens all of them; turning on a channel's full history is now a run of its own, listed under Ad-hoc runs.
|
||||
- Configuration: how long the run history is kept (Activity › Run history, 180 days by default). Stats keeps your own numbers; the system quota view moved to Activity › Quota.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Configuration: saving a setting with the value already in force now keeps it, instead of dropping the change.
|
||||
- Configuration: the unused SponsorBlock default setting is gone.
|
||||
|
||||
## 0.107.3 — 2026-10-17
|
||||
|
||||
### New
|
||||
|
||||
@@ -63,7 +63,7 @@ how it gets built. Concretely:
|
||||
## Architecture shape (don't fight it)
|
||||
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`,
|
||||
`downloads/`, `reader/`, `discover/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell (a shared `/watch` link boots from its own
|
||||
@@ -92,8 +92,8 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` and `jobruns` own their session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / a table of its own (the job intervals had one) |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
|
||||
@@ -630,41 +630,7 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/admin_logs.py": [
|
||||
{
|
||||
"code": "reportReturnType",
|
||||
"range": {
|
||||
"startColumn": 15,
|
||||
"endColumn": 47,
|
||||
"lineCount": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/audit.py": [
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 17,
|
||||
"endColumn": 98,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportAssignmentType",
|
||||
"range": {
|
||||
"startColumn": 17,
|
||||
"endColumn": 98,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 22,
|
||||
"endColumn": 97,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportAttributeAccessIssue",
|
||||
"range": {
|
||||
@@ -726,162 +692,6 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/downloads.py": [
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 17,
|
||||
"endColumn": 19,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 82,
|
||||
"endColumn": 84,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 87,
|
||||
"endColumn": 89,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 30,
|
||||
"endColumn": 32,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 49,
|
||||
"endColumn": 54,
|
||||
"lineCount": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/playlists.py": [
|
||||
{
|
||||
"code": "reportOptionalOperand",
|
||||
"range": {
|
||||
"startColumn": 8,
|
||||
"endColumn": 9,
|
||||
"lineCount": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalOperand",
|
||||
"range": {
|
||||
"startColumn": 8,
|
||||
"endColumn": 9,
|
||||
"lineCount": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 13,
|
||||
"endColumn": 5,
|
||||
"lineCount": 7
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 8,
|
||||
"endColumn": 15,
|
||||
"lineCount": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 16,
|
||||
"endColumn": 5,
|
||||
"lineCount": 11
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 8,
|
||||
"endColumn": 15,
|
||||
"lineCount": 9
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 13,
|
||||
"endColumn": 5,
|
||||
"lineCount": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 8,
|
||||
"endColumn": 15,
|
||||
"lineCount": 3
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 12,
|
||||
"endColumn": 31,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 23,
|
||||
"endColumn": 27,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 12,
|
||||
"endColumn": 34,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 26,
|
||||
"endColumn": 30,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 12,
|
||||
"endColumn": 28,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 23,
|
||||
"endColumn": 27,
|
||||
"lineCount": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/plex/browse.py": [
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
|
||||
@@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config, pool
|
||||
from app.config import settings
|
||||
from app.db import Base
|
||||
from app.logs import configure_logging
|
||||
import app.models # noqa: F401 (import so models register on Base.metadata)
|
||||
import app.tables # noqa: F401 (import so every table registers on Base.metadata)
|
||||
|
||||
config = context.config
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Background runs get a durable history
|
||||
|
||||
Revision ID: 0109_job_runs
|
||||
Revises: 0108_folder_links
|
||||
Create Date: 2026-10-17
|
||||
|
||||
What a scheduler job last did lived in the API process's memory, gone at every restart; the audit
|
||||
log kept a row for a run only when it was manual, failed, or changed something. Now every run is a
|
||||
`job_runs` row, written when it starts and when it ends. A run's quota spend points at it
|
||||
(`quota_events.job_run_id`), and so do the lines it logged (`log_events.run_id`, no foreign key: the
|
||||
log writer runs behind the event).
|
||||
|
||||
Runs are not admin actions, so the audit log stops carrying them: the existing `job_run` audit rows
|
||||
MOVE into `job_runs` (start = end = when the row was written, the only time it kept), and leave the
|
||||
audit log. The downgrade moves every run back into the audit log as such a row (one still running as
|
||||
interrupted, what the audit row has no column for in its `after`), then drops the table.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0109_job_runs"
|
||||
down_revision: str | None = "0108_folder_links"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"job_runs",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True),
|
||||
sa.Column("job", sa.String(40), nullable=False),
|
||||
sa.Column("trigger", sa.String(16), nullable=False),
|
||||
sa.Column("actor_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="SET NULL")),
|
||||
sa.Column("target", sa.String(128)),
|
||||
sa.Column("status", sa.String(16), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("summary", sa.Text()),
|
||||
sa.Column("result", postgresql.JSONB()),
|
||||
sa.Column("error", sa.Text()),
|
||||
sa.Column("failure", postgresql.JSONB()),
|
||||
sa.Column("quota_units", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('running', 'ok', 'error', 'skipped', 'interrupted')",
|
||||
name="ck_job_runs_status",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"trigger IN ('scheduled', 'manual', 'ad_hoc')", name="ck_job_runs_trigger"
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'running') = (finished_at IS NULL)", name="ck_job_runs_finished"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_job_runs_job_started", "job_runs", ["job", "started_at"])
|
||||
op.create_index("ix_job_runs_actor_id", "job_runs", ["actor_id"])
|
||||
op.create_index("ix_job_runs_started_at", "job_runs", ["started_at"])
|
||||
|
||||
op.add_column(
|
||||
"quota_events",
|
||||
sa.Column("job_run_id", sa.BigInteger(), sa.ForeignKey("job_runs.id", ondelete="SET NULL")),
|
||||
)
|
||||
op.create_index("ix_quota_events_job_run_id", "quota_events", ["job_run_id"])
|
||||
|
||||
op.add_column("log_events", sa.Column("run_id", sa.BigInteger()))
|
||||
op.create_index(
|
||||
"ix_log_events_run",
|
||||
"log_events",
|
||||
["run_id"],
|
||||
postgresql_where=sa.text("run_id IS NOT NULL"),
|
||||
)
|
||||
|
||||
# The audit trail's runs move over: the three statuses the scheduler wrote, and `interrupted`,
|
||||
# which a downgrade writes back (as it does a run's trigger, target, spend and start, which the
|
||||
# audit row has no column for: its actor alone cannot say "manual" once that admin is deleted),
|
||||
# so a rollback and a redeploy keep every run as it was. A status the run table does
|
||||
# not know would fail the CHECK.
|
||||
op.execute(
|
||||
"""
|
||||
INSERT INTO job_runs (job, trigger, actor_id, target, status, started_at, finished_at,
|
||||
summary, result, error, failure, quota_units)
|
||||
SELECT target_id,
|
||||
coalesce(after::jsonb ->> 'trigger',
|
||||
CASE WHEN actor_id IS NULL THEN 'scheduled' ELSE 'manual' END),
|
||||
actor_id,
|
||||
after::jsonb ->> 'target',
|
||||
after::jsonb ->> 'status',
|
||||
coalesce((after::jsonb ->> 'started_at')::timestamptz, created_at),
|
||||
created_at,
|
||||
CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb ->> 'summary' END,
|
||||
CASE WHEN after::jsonb ->> 'status' <> 'error' THEN after::jsonb -> 'result' END,
|
||||
CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb ->> 'summary' END,
|
||||
CASE WHEN after::jsonb ->> 'status' = 'error' THEN after::jsonb -> 'result' END,
|
||||
coalesce((after::jsonb ->> 'quota_units')::int, 0)
|
||||
FROM audit_log
|
||||
WHERE action = 'job_run'
|
||||
AND target_id IS NOT NULL
|
||||
AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted')
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
DELETE FROM audit_log
|
||||
WHERE action = 'job_run'
|
||||
AND target_id IS NOT NULL
|
||||
AND after::jsonb ->> 'status' IN ('ok', 'error', 'skipped', 'interrupted')
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# A run still `running` is one the rollback stopped: it goes back as interrupted. What the audit
|
||||
# row's own columns cannot say rides in `after` (its trigger, target, spend and start), so the
|
||||
# upgrade restores them.
|
||||
op.execute(
|
||||
"""
|
||||
INSERT INTO audit_log (actor_id, action, target_type, target_id, summary, after, created_at)
|
||||
SELECT actor_id,
|
||||
'job_run',
|
||||
'job',
|
||||
job,
|
||||
left(job || ': ' || st
|
||||
|| coalesce(' — ' || CASE WHEN st = 'error' THEN error ELSE summary END, ''),
|
||||
255),
|
||||
(jsonb_build_object(
|
||||
'status', st,
|
||||
'summary', CASE WHEN st = 'error' THEN error ELSE summary END)
|
||||
|| CASE
|
||||
WHEN st = 'error' AND failure IS NOT NULL
|
||||
THEN jsonb_build_object('result', failure)
|
||||
WHEN st <> 'error' AND result IS NOT NULL
|
||||
THEN jsonb_build_object('result', result)
|
||||
ELSE '{}'::jsonb
|
||||
END
|
||||
|| jsonb_build_object('trigger', trigger, 'quota_units', units,
|
||||
'started_at', started_at)
|
||||
|| CASE WHEN target IS NOT NULL
|
||||
THEN jsonb_build_object('target', target) ELSE '{}'::jsonb END)::json,
|
||||
coalesce(finished_at, started_at)
|
||||
FROM (SELECT *, CASE WHEN status = 'running' THEN 'interrupted' ELSE status END AS st,
|
||||
-- A run stopped mid-way has not summed its spend yet: sum it as the startup
|
||||
-- sweep would, before the link to it is dropped.
|
||||
CASE WHEN status = 'running'
|
||||
THEN (SELECT coalesce(sum(q.units), 0) FROM quota_events q
|
||||
WHERE q.job_run_id = job_runs.id)
|
||||
ELSE quota_units END AS units
|
||||
FROM job_runs) AS r
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
op.drop_index("ix_log_events_run", table_name="log_events")
|
||||
op.drop_column("log_events", "run_id")
|
||||
op.drop_index("ix_quota_events_job_run_id", table_name="quota_events")
|
||||
op.drop_column("quota_events", "job_run_id")
|
||||
op.drop_index("ix_job_runs_started_at", table_name="job_runs")
|
||||
op.drop_index("ix_job_runs_actor_id", table_name="job_runs")
|
||||
op.drop_index("ix_job_runs_job_started", table_name="job_runs")
|
||||
op.drop_table("job_runs")
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Job intervals move into sysconfig; the unread SponsorBlock default goes
|
||||
|
||||
Revision ID: 0110_intervals_in_sysconfig
|
||||
Revises: 0109_job_runs
|
||||
Create Date: 2026-10-18
|
||||
|
||||
An admin's interval override lived in its own table, `scheduler_settings`, beside the one store every
|
||||
other admin knob uses. Each override becomes the `job_interval_<job>` sysconfig key and the table
|
||||
goes. Only the jobs that still exist are copied: a row for any other id never did anything.
|
||||
|
||||
The `sponsorblock_default` setting was offered on the Configuration page but nothing ever read it, so
|
||||
its key is gone and its stored value with it — the downgrade cannot bring that value back.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0110_intervals_in_sysconfig"
|
||||
down_revision: str | None = "0109_job_runs"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
# The interval jobs as of this revision (a migration does not read the live code's list).
|
||||
_JOBS = (
|
||||
"rss_poll",
|
||||
"enrich",
|
||||
"backfill",
|
||||
"autotag",
|
||||
"shorts",
|
||||
"subscriptions",
|
||||
"playlist_sync",
|
||||
"maintenance",
|
||||
"explore_cleanup",
|
||||
"download_gc",
|
||||
"plex_sync",
|
||||
"plex_watch_sync",
|
||||
"plex_watch_reconcile",
|
||||
"tmdb_enrich",
|
||||
"people_enrich",
|
||||
"audit_gc",
|
||||
"log_gc",
|
||||
)
|
||||
_PREFIX = "job_interval_"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
sa.text(
|
||||
"INSERT INTO system_config (key, value, encrypted)"
|
||||
" SELECT :prefix || job_id, interval_minutes::text, false FROM scheduler_settings"
|
||||
" WHERE interval_minutes IS NOT NULL AND job_id IN :jobs"
|
||||
" ON CONFLICT (key) DO NOTHING"
|
||||
).bindparams(sa.bindparam("jobs", expanding=True), prefix=_PREFIX, jobs=list(_JOBS))
|
||||
)
|
||||
op.drop_table("scheduler_settings")
|
||||
op.execute("DELETE FROM system_config WHERE key = 'sponsorblock_default'")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.create_table(
|
||||
"scheduler_settings",
|
||||
sa.Column("job_id", sa.String(40), primary_key=True),
|
||||
sa.Column("interval_minutes", sa.Integer(), nullable=False),
|
||||
)
|
||||
op.execute(
|
||||
sa.text(
|
||||
"INSERT INTO scheduler_settings (job_id, interval_minutes)"
|
||||
" SELECT substr(key, :start), value::int FROM system_config WHERE key IN :keys"
|
||||
).bindparams(
|
||||
sa.bindparam("keys", expanding=True),
|
||||
start=len(_PREFIX) + 1,
|
||||
keys=[_PREFIX + j for j in _JOBS],
|
||||
)
|
||||
)
|
||||
op.execute(
|
||||
sa.text("DELETE FROM system_config WHERE key IN :keys").bindparams(
|
||||
sa.bindparam("keys", expanding=True), keys=[_PREFIX + j for j in _JOBS]
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,28 @@
|
||||
"""An audit row keeps the request that made it
|
||||
|
||||
Revision ID: 0111_audit_request_id
|
||||
Revises: 0110_intervals_in_sysconfig
|
||||
Create Date: 2026-10-18
|
||||
|
||||
The activity page opens an admin action's log lines from its row: `audit_log.request_id` is the
|
||||
request the change was made in, the same id `log_events.request_id` carries. Older rows have none —
|
||||
the request that made them was never recorded.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0111_audit_request_id"
|
||||
down_revision: str | None = "0110_intervals_in_sysconfig"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("audit_log", sa.Column("request_id", sa.String(64)))
|
||||
op.create_index("ix_audit_log_request_id", "audit_log", ["request_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_audit_log_request_id", table_name="audit_log")
|
||||
op.drop_column("audit_log", "request_id")
|
||||
@@ -21,7 +21,8 @@ Three layers, by *who* runs the code — not by feature:
|
||||
`issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker.
|
||||
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth` (with `apitokens`, a machine's
|
||||
folder-scoped way into the Files library), `sysconfig`, `quota`, `audit`,
|
||||
folder-scoped way into the Files library), `sysconfig`, `quota`, `audit`, `jobruns` (every
|
||||
background run's durable row, its words in `runstatus`),
|
||||
`notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
||||
@@ -36,7 +37,7 @@ pre-abstract for a single caller; don't leave shared logic duplicated once there
|
||||
## Transaction ownership
|
||||
|
||||
This follows CLAUDE.md's canonical rule verbatim — **"the handler/job-wrapper commits; helpers flush
|
||||
(only `quota` owns its session)"** — so read it that way, not as an exception to it. The commit belongs
|
||||
(only `quota` and `jobruns` own their session)"** — so read it that way, not as an exception to it. The commit belongs
|
||||
to the **operation owner**; anything called *inside* another transaction only stages its change
|
||||
(`db.add` / `flush`) and lets the owner commit, so the whole operation is one transaction.
|
||||
`audit.record()` is the reference: it adds the audit row and the caller commits it together with the
|
||||
@@ -45,7 +46,7 @@ mutation it records, so an audit entry can never exist without its change (or vi
|
||||
- The **operation owner** is usually the route handler. A `service.*` operation the handler delegates a
|
||||
whole transition to (e.g. `service.cancel`) *is* that owner and commits its one transition — it isn't
|
||||
a mid-operation helper. A helper that owns its OWN short-lived session (the worker's `jobdb` writes,
|
||||
the same carve-out CLAUDE.md grants `quota`) commits that session. Everything else flushes.
|
||||
`jobruns`' run rows — the same carve-out CLAUDE.md grants `quota`) commits that session. Everything else flushes.
|
||||
- `SessionLocal` is `expire_on_commit=False`: after a raw `UPDATE` + commit, an ORM object you already
|
||||
loaded is **stale**. Read the fresh value from the row (RETURNING, or a re-`SELECT`), not the ORM
|
||||
attribute — see `service.decr_ref_count`.
|
||||
@@ -67,8 +68,9 @@ mutation it records, so an audit entry can never exist without its change (or vi
|
||||
- **`app.config.settings`** — boot/env config (DB URL, download root, worker concurrency): read once at
|
||||
startup, not user-tunable at runtime.
|
||||
- **`sysconfig.SPECS`** — the runtime admin knobs (DB-override-or-default). A new admin-tunable value
|
||||
goes in `SPECS`; don't invent a fourth store. (`state.py`'s `maintenance_batch` predates this and is
|
||||
slated to fold into `sysconfig` — do not add more like it.)
|
||||
goes in `SPECS`; don't invent a fourth store. Each spec states when a change takes effect
|
||||
(`applies`, required): `live` keys are read where they are used, never at import (test-held). The
|
||||
scheduler's job intervals are `SPECS` keys too, edited on the Scheduler page, not the Configuration page.
|
||||
|
||||
## The gate
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
"""Admin/scheduler audit trail: an append-only record of who changed what and when.
|
||||
"""Admin audit trail: an append-only record of who changed what and when.
|
||||
|
||||
Design (locked): log at ACTION / job-run granularity, NEVER per-row — a scheduler run that
|
||||
touches thousands of videos is ONE audit row summarising the outcome. Automatic job runs are
|
||||
only logged when they actually changed something (or errored); all admin actions and all manual
|
||||
runs are always logged. Secrets are never stored (config changes log the key name only).
|
||||
Design (locked): log at ACTION granularity, NEVER per-row — an action that touches thousands of
|
||||
rows is ONE audit row summarising the outcome. Background runs are not actions: they have their
|
||||
own history (app/jobruns.py). Secrets are never stored (config changes log the key name only).
|
||||
Each row keeps the request that made it, so the request's log lines are one click away.
|
||||
|
||||
`record()` only db.add()s the row — the caller commits it in the SAME transaction as the
|
||||
mutation it describes, so the trail can't drift from reality (and a rolled-back mutation drops
|
||||
@@ -15,6 +15,7 @@ from datetime import datetime, timedelta, timezone
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import AuditLog
|
||||
|
||||
|
||||
@@ -48,8 +49,6 @@ class AuditAction:
|
||||
MAINTENANCE_TUNE = "maintenance_tune"
|
||||
SYNC_PAUSE = "sync_pause"
|
||||
SYNC_RESUME = "sync_resume"
|
||||
# Scheduler job outcome / run-summary (actor_id set = a manual "run now"; NULL = automatic)
|
||||
JOB_RUN = "job_run"
|
||||
# Per-user download cookie/auth credential set or removed (the value is NEVER recorded)
|
||||
DOWNLOAD_CREDENTIAL_SET = "download_credential_set"
|
||||
DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete"
|
||||
@@ -104,6 +103,7 @@ def record(
|
||||
summary=(summary or None) and summary[:255],
|
||||
before=before,
|
||||
after=after,
|
||||
request_id=logs.current_request_id(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -268,10 +268,12 @@ class Settings(BaseSettings):
|
||||
# YouTube quota history (quota_events): days kept (0 = forever). A year, so the admin usage view's
|
||||
# longest window (365 days) stays whole; the same daily cleanup job prunes it.
|
||||
quota_retention_days: int = 365
|
||||
# Background-run history (job_runs): days kept (0 = forever). The audit log's 180, which is how
|
||||
# long a run lived there before it moved here: ~1.8k rows a day at the default intervals
|
||||
# (measured 2026-10-17), ~320k at the limit. The same daily cleanup job prunes it.
|
||||
job_run_retention_days: int = 180
|
||||
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
||||
download_layout: DownloadLayout = "plex"
|
||||
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
||||
sponsorblock_default: bool = False
|
||||
|
||||
# --- Diagnostics / logging ---
|
||||
# BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at
|
||||
|
||||
@@ -22,7 +22,8 @@ to the client, which has both the native payload and the translations.
|
||||
"""
|
||||
|
||||
from app.discover.budget import remaining_seconds
|
||||
from app.models import Channel, Playlist
|
||||
from app.models import Channel
|
||||
from app.reader.models import Playlist
|
||||
|
||||
# Plex artwork is a poster for the things you browse and a landscape still for the things you play.
|
||||
_POSTER_KINDS = ("movie", "show")
|
||||
|
||||
@@ -22,10 +22,12 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.discover import cards, queries
|
||||
from app.models import Channel, ComfortPin, PlexItem, PlexShow, Playlist, User, Video
|
||||
from app.models import Channel, PlexItem, PlexShow, User, Video
|
||||
from app.reader.models import ComfortPin, Playlist
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import SHOW_NEW, leaf_card, show_card
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
# kind -> the ComfortPin column that holds it. The client's vocabulary and the schema's, in one place.
|
||||
PIN_COLUMNS: dict[str, str] = {
|
||||
@@ -92,7 +94,7 @@ def reorder(db: Session, user: User, ids: list[int]) -> None:
|
||||
# old slots — and one global lock order means two reorders cannot deadlock.
|
||||
locked = db.execute(
|
||||
select(ComfortPin)
|
||||
.where(ComfortPin.user_id == user.id)
|
||||
.where(owned_by(ComfortPin.user_id, user.id))
|
||||
.order_by(ComfortPin.id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
@@ -117,7 +119,7 @@ def list_pins(db: Session, user: User) -> list[ComfortPin]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(ComfortPin)
|
||||
.where(ComfortPin.user_id == user.id)
|
||||
.where(owned_by(ComfortPin.user_id, user.id))
|
||||
.order_by(ComfortPin.position.asc(), ComfortPin.id.asc())
|
||||
)
|
||||
.scalars()
|
||||
@@ -132,17 +134,44 @@ def next_position(db: Session, user: User) -> int:
|
||||
deletes the row, and every FK cascade silently removes a pin when the thing it points at goes
|
||||
away — so past the first hole a count under-counts and would drop the new pin into the MIDDLE of
|
||||
the shelf, or onto a position another pin already holds (nothing in the schema forbids that).
|
||||
Same idiom as `routes/playlists._next_playlist_position`."""
|
||||
Same idiom as a new playlist's place in the rail (`reader.service`)."""
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.max(ComfortPin.position), -1)).where(
|
||||
ComfortPin.user_id == user.id
|
||||
owned_by(ComfortPin.user_id, user.id)
|
||||
)
|
||||
)
|
||||
+ 1
|
||||
)
|
||||
|
||||
|
||||
def own_pin(db: Session, user: User, pin_id: int) -> ComfortPin | None:
|
||||
return owned_row(db, ComfortPin, pin_id, user)
|
||||
|
||||
|
||||
def pin_of(db: Session, user: User, column: str, value: object) -> ComfortPin | None:
|
||||
"""The user's pin of this thing (`column`/`value` from `resolve_ref`), if it is on the shelf."""
|
||||
return db.execute(
|
||||
select(ComfortPin).where(
|
||||
owned_by(ComfortPin.user_id, user), getattr(ComfortPin, column) == value
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def add(db: Session, user: User, column: str, value: object) -> ComfortPin:
|
||||
"""A new pin at the end of the shelf. A second pin of the same thing (another tab, at the same
|
||||
moment) raises `IntegrityError` at the flush: the unique constraint is the guard."""
|
||||
pin = ComfortPin(user_id=user.id, position=next_position(db, user), **{column: value})
|
||||
db.add(pin)
|
||||
db.flush()
|
||||
return pin
|
||||
|
||||
|
||||
def remove(db: Session, pin: ComfortPin) -> None:
|
||||
db.delete(pin)
|
||||
db.flush()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Batch:
|
||||
"""What the shelf resolves for every pinned channel and playlist in one go."""
|
||||
|
||||
@@ -29,19 +29,12 @@ from app.feedquery import (
|
||||
serialize,
|
||||
sort_keys,
|
||||
)
|
||||
from app.models import (
|
||||
Channel,
|
||||
PlaylistItem,
|
||||
PlexItem,
|
||||
PlexShow,
|
||||
PlexState,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.models import Channel, PlexItem, PlexShow, PlexState, User, Video
|
||||
from app.reader.models import PlaylistItem, VideoState
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||
from app.plex.query import apply_show_state_filter, content_libs
|
||||
from app.userscope import owned_by
|
||||
|
||||
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
|
||||
# (`routes/plex/discover.py` imports it): the two surfaces should feel like the same app, and a rail
|
||||
@@ -209,7 +202,7 @@ def _unseen_movie_conds(uid: int):
|
||||
.select_from(PlexState)
|
||||
.where(
|
||||
PlexState.item_id == PlexItem.id,
|
||||
PlexState.user_id == uid,
|
||||
owned_by(PlexState.user_id, uid),
|
||||
or_(
|
||||
PlexState.status.in_(("watched", "hidden")),
|
||||
# Started counts as "not a surprise" too — see the note in `yt_surprise`.
|
||||
@@ -237,7 +230,9 @@ def plex_in_progress(
|
||||
if movie_libs:
|
||||
mq = (
|
||||
select(PlexItem, PlexState)
|
||||
.join(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
|
||||
.join(
|
||||
PlexState, and_(PlexState.item_id == PlexItem.id, owned_by(PlexState.user_id, uid))
|
||||
)
|
||||
.where(
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.library_id.in_(movie_libs),
|
||||
@@ -331,7 +326,9 @@ def next_episode(db: Session, uid: int, show_id: int, *, lang: str) -> dict | No
|
||||
)
|
||||
base = (
|
||||
select(PlexItem, PlexState)
|
||||
.outerjoin(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
|
||||
.outerjoin(
|
||||
PlexState, and_(PlexState.item_id == PlexItem.id, owned_by(PlexState.user_id, uid))
|
||||
)
|
||||
.where(PlexItem.show_id == show_id, PlexItem.kind == "episode")
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
@@ -22,7 +22,7 @@ from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
from app.downloads.models import MediaAsset
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
@@ -33,8 +33,9 @@ from app import logs
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential, MediaAsset
|
||||
from app.downloads.models import DownloadCredential, MediaAsset
|
||||
from app.security import decrypt_optional, encrypt
|
||||
from app.userscope import owned_by
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
@@ -157,7 +158,7 @@ def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
||||
def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None:
|
||||
return db.execute(
|
||||
select(DownloadCredential).where(
|
||||
DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key
|
||||
owned_by(DownloadCredential.user_id, user_id), DownloadCredential.host_key == host_key
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
@@ -214,7 +215,7 @@ def status_list(db: Session, user_id: int) -> list[dict]:
|
||||
have = {
|
||||
c.host_key: c
|
||||
for c in db.execute(
|
||||
select(DownloadCredential).where(DownloadCredential.user_id == user_id)
|
||||
select(DownloadCredential).where(owned_by(DownloadCredential.user_id, user_id))
|
||||
).scalars()
|
||||
}
|
||||
out = []
|
||||
|
||||
@@ -32,7 +32,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, FileFolder, MediaAsset
|
||||
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||
from app.userscope import owned_by
|
||||
|
||||
|
||||
class FolderKind(StrEnum):
|
||||
@@ -106,7 +107,9 @@ def source_key_for(db: Session, job: DownloadJob) -> str:
|
||||
def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
|
||||
"""The user's source folder for `key`, made on first use. A race on the unique (user, key) is
|
||||
settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
|
||||
stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
|
||||
stmt = select(FileFolder).where(
|
||||
owned_by(FileFolder.user_id, user_id), FileFolder.source_key == key
|
||||
)
|
||||
folder = db.execute(stmt).scalar_one_or_none()
|
||||
if folder is not None:
|
||||
return folder
|
||||
@@ -131,7 +134,7 @@ def hub_folder(db: Session, user_id: int) -> FileFolder:
|
||||
"""The user's Hub folder, made on first use; a race on its one-per-user index is settled like
|
||||
`source_folder`'s."""
|
||||
stmt = select(FileFolder).where(
|
||||
FileFolder.user_id == user_id, FileFolder.kind == FolderKind.HUB
|
||||
owned_by(FileFolder.user_id, user_id), FileFolder.kind == FolderKind.HUB
|
||||
)
|
||||
folder = db.execute(stmt).scalar_one_or_none()
|
||||
if folder is not None:
|
||||
@@ -310,3 +313,28 @@ def delete_own(db: Session, folder: FileFolder) -> int:
|
||||
db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete
|
||||
db.delete(folder)
|
||||
return len(jobs)
|
||||
|
||||
|
||||
def create_own(db: Session, user_id: int, name: str) -> FileFolder:
|
||||
"""A new folder of the user's own; FolderError("file_folder_name_taken") when they already have
|
||||
one by that name (the unique (user, lower(name)))."""
|
||||
folder = FileFolder(user_id=user_id, kind=FolderKind.OWN, name=name)
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(folder)
|
||||
db.flush()
|
||||
except IntegrityError as e:
|
||||
raise FolderError("file_folder_name_taken") from e
|
||||
return folder
|
||||
|
||||
|
||||
def rename(db: Session, folder: FileFolder, name: str) -> None:
|
||||
"""Rename one of the user's own folders (a source folder's name is its origin's)."""
|
||||
if folder.kind != FolderKind.OWN:
|
||||
raise FolderError("file_folder_not_own")
|
||||
folder.name = name
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.flush()
|
||||
except IntegrityError as e:
|
||||
raise FolderError("file_folder_name_taken") from e
|
||||
|
||||
@@ -31,7 +31,7 @@ from app import apitokens, logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
@@ -6,7 +6,7 @@ from sqlalchemy import text
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
# --- small DB helpers (short-lived sessions; the download itself holds no transaction) -------
|
||||
|
||||
@@ -15,20 +15,24 @@ grant appended to the file URL, so the raw password never rides in the media URL
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import update as sql_update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import security
|
||||
from app import audit, security
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
from app.models import User
|
||||
from app.utils import iso
|
||||
from app.userscope import owned_by
|
||||
|
||||
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
|
||||
# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files.
|
||||
# MAX_LIST`; a test holds the two equal — this layer may not import that one).
|
||||
# The most files a folder link lists: as many as the owner's own Files list shows
|
||||
# (`queries.MAX_LIST`; a test holds the two equal — `queries` imports this module).
|
||||
FOLDER_MAX = 500
|
||||
|
||||
|
||||
@@ -59,7 +63,7 @@ def in_folder(folder: FileFolder) -> tuple:
|
||||
moved out stops opening through the link at its next request; one moved in starts."""
|
||||
return (
|
||||
DownloadJob.folder_id == folder.id,
|
||||
DownloadJob.user_id == folder.user_id,
|
||||
owned_by(DownloadJob.user_id, folder.user_id),
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
)
|
||||
|
||||
@@ -170,3 +174,117 @@ def public_meta(
|
||||
"file_url": file_url,
|
||||
"poster_url": poster_url,
|
||||
}
|
||||
|
||||
|
||||
# --- the owner's writes --------------------------------------------------------------------
|
||||
# Each flushes and records its audit row; the handler commits.
|
||||
|
||||
|
||||
def expiry(days: int | None) -> datetime | None:
|
||||
"""None/0/negative → no expiry; otherwise N days out, capped at ~10 years."""
|
||||
if not days or days <= 0:
|
||||
return None
|
||||
return datetime.now(timezone.utc) + timedelta(days=min(days, 3650))
|
||||
|
||||
|
||||
def audit_view(link: DownloadLink) -> dict:
|
||||
"""A link's settings as the audit records them: whether it has a password, never which."""
|
||||
return {
|
||||
"allow_download": link.allow_download,
|
||||
"has_password": link.password_hash is not None,
|
||||
"expires_at": iso(link.expires_at),
|
||||
}
|
||||
|
||||
|
||||
def create(
|
||||
db: Session,
|
||||
user: User,
|
||||
*,
|
||||
password: str,
|
||||
allow_download: bool,
|
||||
expires_days: int | None,
|
||||
**target: int,
|
||||
) -> tuple[DownloadLink, str]:
|
||||
"""A link to `target` (`job_id=` or `folder_id=`, already checked to be the user's), and the
|
||||
token it is opened by — which is in hand only now."""
|
||||
token = new_token()
|
||||
link = DownloadLink(
|
||||
**target,
|
||||
created_by=user.id,
|
||||
allow_download=allow_download,
|
||||
password_hash=security.hash_password(password) if password else None,
|
||||
expires_at=expiry(expires_days),
|
||||
)
|
||||
seal(link, token)
|
||||
db.add(link)
|
||||
db.flush()
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_CREATE,
|
||||
target_type="download_link",
|
||||
target_id=link.id,
|
||||
after={**target, **audit_view(link)},
|
||||
)
|
||||
return link, token
|
||||
|
||||
|
||||
def update(
|
||||
db: Session,
|
||||
user: User,
|
||||
link: DownloadLink,
|
||||
*,
|
||||
allow_download: bool | None = None,
|
||||
expires_days: int | None = None,
|
||||
password: str | None = None,
|
||||
sent: frozenset[str] = frozenset(),
|
||||
) -> None:
|
||||
"""Change what the caller sent (`sent` names the fields, so `expires_days=None` can mean "never
|
||||
expires"). Setting, clearing or changing the password bumps its version, so any grant issued
|
||||
under the previous one stops working now rather than living out its TTL."""
|
||||
before = audit_view(link)
|
||||
if "allow_download" in sent:
|
||||
link.allow_download = bool(allow_download)
|
||||
if "expires_days" in sent:
|
||||
link.expires_at = expiry(expires_days)
|
||||
if "password" in sent:
|
||||
pw = (password or "").strip()
|
||||
link.password_hash = security.hash_password(pw) if pw else None
|
||||
link.password_version += 1
|
||||
after = audit_view(link)
|
||||
if after != before or "password" in sent:
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_UPDATE,
|
||||
target_type="download_link",
|
||||
target_id=link.id,
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def revoke(db: Session, user: User, link: DownloadLink) -> None:
|
||||
link_id = link.id
|
||||
db.delete(link)
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_REVOKE,
|
||||
target_type="download_link",
|
||||
target_id=link_id,
|
||||
after={"job_id": link.job_id} if link.job_id is not None else {"folder_id": link.folder_id},
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def count_view(db: Session, link: DownloadLink) -> None:
|
||||
"""One more opening of the link. In SQL, not `+= 1` on the loaded row: two opens at once would
|
||||
both write the same count."""
|
||||
db.execute(
|
||||
sql_update(DownloadLink)
|
||||
.where(DownloadLink.id == link.id)
|
||||
.values(view_count=DownloadLink.view_count + 1)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
"""Capture's tables: the downloaded files (`MediaAsset`), each user's jobs over them and the folders
|
||||
they sit in, the shares and public links that hand them on, a user's quota, cookies and format
|
||||
profiles, and where they stopped playing one. Only this unit writes them; another unit reads them
|
||||
along a declared edge, or asks this unit's service."""
|
||||
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
Date,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, validates
|
||||
|
||||
from app.db import Base
|
||||
from app.models import DISPLAY_TEXT_MAX, TimestampMixin, UpdatedAtMixin
|
||||
|
||||
|
||||
class LibraryState(Base):
|
||||
"""Where a user stopped in a library item (their own download, or one shared with them) — the
|
||||
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
|
||||
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
|
||||
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
|
||||
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
|
||||
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
|
||||
|
||||
__tablename__ = "library_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class MediaAsset(Base, TimestampMixin):
|
||||
"""A physical downloaded media file — the SHARED cache layer of the download center.
|
||||
|
||||
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
|
||||
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
|
||||
the same video with a format that hashes to the same `format_sig`, their `download_job`
|
||||
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
||||
shared; per-user edited derivatives (phase 2) are never cached here.
|
||||
|
||||
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
|
||||
system-decided and bound to the source id — never renamed (the user's custom name is display
|
||||
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
||||
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
|
||||
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
|
||||
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
|
||||
|
||||
__tablename__ = "media_assets"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
|
||||
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="pending", server_default="pending", index=True
|
||||
) # "pending" | "ready" | "error"
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
|
||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
||||
# card / watch page / link preview can still show an image. Relative to files_root.
|
||||
poster_path: Mapped[str | None] = mapped_column(Text)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||
width: Mapped[int | None] = mapped_column(Integer)
|
||||
height: Mapped[int | None] = mapped_column(Integer)
|
||||
container: Mapped[str | None] = mapped_column(String(16))
|
||||
vcodec: Mapped[str | None] = mapped_column(String(32))
|
||||
acodec: Mapped[str | None] = mapped_column(String(32))
|
||||
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
|
||||
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
|
||||
# still pending (kind is known once the file is probed at finalize). It is the single stored home
|
||||
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
|
||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||
title: Mapped[str | None] = mapped_column(Text)
|
||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
|
||||
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
|
||||
uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
upload_date: Mapped[date | None] = mapped_column(Date)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
||||
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
||||
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
|
||||
# fall back to a URL derived from source_kind+source_ref).
|
||||
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
|
||||
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
|
||||
sha256: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
@validates("rel_path")
|
||||
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
|
||||
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
|
||||
self.sha256 = None
|
||||
return rel_path
|
||||
|
||||
|
||||
class DownloadProfile(Base, TimestampMixin):
|
||||
"""A named download preset — a reusable format profile.
|
||||
|
||||
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
|
||||
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
|
||||
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
|
||||
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
|
||||
`format_sig`."""
|
||||
|
||||
__tablename__ = "download_profiles"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
) # NULL for a builtin preset
|
||||
name: Mapped[str] = mapped_column(String(80))
|
||||
spec: Mapped[dict] = mapped_column(JSON)
|
||||
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class FileFolder(Base, TimestampMixin):
|
||||
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
|
||||
|
||||
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
|
||||
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
|
||||
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
|
||||
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
|
||||
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
|
||||
|
||||
__tablename__ = "file_folders"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||
Index(
|
||||
"uq_file_folder_own_name",
|
||||
"user_id",
|
||||
func.lower(text("name")),
|
||||
unique=True,
|
||||
postgresql_where=text("name IS NOT NULL"),
|
||||
),
|
||||
# One Hub folder per user.
|
||||
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
|
||||
CheckConstraint(
|
||||
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
||||
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
|
||||
name="ck_file_folder_kind",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
|
||||
# "site:" + a host name (DNS caps one at 253 characters).
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
retention_days: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
|
||||
def _ready_on_insert(context) -> datetime | None:
|
||||
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
|
||||
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
|
||||
done = context.get_current_parameters().get("status") == "done"
|
||||
return datetime.now(timezone.utc) if done else None
|
||||
|
||||
|
||||
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||
|
||||
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
|
||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
|
||||
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
|
||||
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
|
||||
`display_name` is the user's own name for the file (display + the Content-Disposition on
|
||||
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
|
||||
(SET NULL if the asset is GC'd — the job then shows as expired).
|
||||
|
||||
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
|
||||
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
|
||||
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
|
||||
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
|
||||
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
|
||||
|
||||
__tablename__ = "download_jobs"
|
||||
__table_args__ = (
|
||||
Index("ix_download_jobs_claim", "status", "queue_pos"),
|
||||
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
|
||||
CheckConstraint(
|
||||
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512))
|
||||
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
|
||||
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
|
||||
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
|
||||
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
||||
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
||||
source_job_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="SET NULL")
|
||||
)
|
||||
source_asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL")
|
||||
)
|
||||
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
|
||||
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
|
||||
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
|
||||
edit_spec: Mapped[dict | None] = mapped_column(JSON)
|
||||
profile_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||
)
|
||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
||||
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
||||
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
extra_links: Mapped[list | None] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="queued", server_default="queued", index=True
|
||||
) # queued | running | paused | done | error | canceled
|
||||
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
|
||||
eta_s: Mapped[int | None] = mapped_column(Integer)
|
||||
phase: Mapped[str | None] = mapped_column(String(32))
|
||||
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
|
||||
error_code: Mapped[str | None] = mapped_column(
|
||||
String(24)
|
||||
) # category, see downloads.errors.classify
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
|
||||
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
|
||||
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
|
||||
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
||||
)
|
||||
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
|
||||
# traces its origin through its parent, and that parent may be deleted or expire later — the
|
||||
# file must still know which source folder it may go back to. NULL only for a file placed before
|
||||
# the column existed outside a source folder; `folders.source_key_for` works those out.
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# When the file became done — what the library's "Added" means: a download that failed for days
|
||||
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
|
||||
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
|
||||
ready_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), default=_ready_on_insert
|
||||
)
|
||||
|
||||
|
||||
class DownloadShare(Base, TimestampMixin):
|
||||
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
|
||||
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
|
||||
|
||||
__tablename__ = "download_shares"
|
||||
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
|
||||
from_user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
|
||||
class DownloadQuota(Base, UpdatedAtMixin):
|
||||
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
|
||||
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
|
||||
byte cap entirely (e.g. for the admin's own account)."""
|
||||
|
||||
__tablename__ = "download_quotas"
|
||||
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
max_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
max_concurrent: Mapped[int] = mapped_column(Integer)
|
||||
max_jobs: Mapped[int] = mapped_column(Integer)
|
||||
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
class DownloadLink(Base, TimestampMixin):
|
||||
"""A public capability link to one download, or to one of its owner's own folders — the "share
|
||||
by link" surface.
|
||||
|
||||
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
|
||||
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
|
||||
the folder at the time it is opened. The token itself is never stored as written: a link is
|
||||
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
|
||||
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
|
||||
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
|
||||
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
|
||||
grants a *registered* user access via the in-app "Shared with me" list."""
|
||||
|
||||
__tablename__ = "download_links"
|
||||
__table_args__ = (
|
||||
# A link opens exactly one thing: a file or a folder.
|
||||
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
password_hash: Mapped[str | None] = mapped_column(Text)
|
||||
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
|
||||
# password immediately invalidates any grants issued under the old one (instead of them living
|
||||
# out the 6h grant TTL).
|
||||
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
|
||||
|
||||
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
|
||||
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
|
||||
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
|
||||
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
|
||||
the routes are write-only (host + updated_at only). See `downloads.credentials`.
|
||||
|
||||
Two load-bearing security controls live around this row, NOT on it:
|
||||
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
|
||||
user id, so the authed asset is private to that user and can never be cache-served to
|
||||
another user (`downloads.credentials.cookie_salt`).
|
||||
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
|
||||
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
|
||||
can't exfiltrate the user's session to a look-alike host.
|
||||
|
||||
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
|
||||
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
|
||||
|
||||
__tablename__ = "download_credentials"
|
||||
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
host_key: Mapped[str] = mapped_column(String(64))
|
||||
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
@@ -16,7 +16,7 @@ from sqlalchemy.orm import Session
|
||||
from app.config import settings
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
|
||||
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
|
||||
|
||||
|
||||
@@ -28,7 +28,8 @@ from app.downloads.jobdb import (
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem
|
||||
from app.models import DISPLAY_TEXT_MAX, PlexItem
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.plex import lookups as plex_lookups
|
||||
from app.plex import paths as plex_paths
|
||||
from app.notifications import create_notification
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
"""Capture's reads: what the library, the download tray, the share dialogs and the public watch pages
|
||||
show. Each returns rows or plain values and writes nothing; whose a row is goes through
|
||||
`userscope.owned_by` / `owned_row`, so a row of someone else's reads as a missing one.
|
||||
|
||||
The HTTP layer (`routes.downloads`, `routes.files`, `routes.public`) asks here and serializes; the
|
||||
writes are the service's (`service`, `folders`, `links`, `quota`)."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.apitokens import FilesCaller, Right
|
||||
from app.collation import collated
|
||||
from app.downloads import folders, links, quota
|
||||
from app.downloads.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
DownloadProfile,
|
||||
DownloadQuota,
|
||||
DownloadShare,
|
||||
FileFolder,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
)
|
||||
from app.downloads.status import FINISHED, JobStatus
|
||||
from app.models import User
|
||||
from app.userscope import owned_by, owned_row
|
||||
from app.utils import like_escape
|
||||
|
||||
# Cap a listing so a heavy library can't return an unbounded result set (the old Download Center's
|
||||
# own cap; far beyond any real library).
|
||||
MAX_LIST = 500
|
||||
|
||||
|
||||
# --- jobs and their files ------------------------------------------------------------------
|
||||
|
||||
|
||||
def own_job(db: Session, user: User, job_id: int) -> DownloadJob | None:
|
||||
return owned_row(db, DownloadJob, job_id, user)
|
||||
|
||||
|
||||
def job(db: Session, job_id: int) -> DownloadJob | None:
|
||||
"""A job by id, whoever's — for a public link, which already names its job."""
|
||||
return db.get(DownloadJob, job_id)
|
||||
|
||||
|
||||
def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob | None:
|
||||
"""A file the caller may read: their own, or one shared with them."""
|
||||
row = db.get(DownloadJob, job_id)
|
||||
if row is None:
|
||||
return None
|
||||
if row.user_id == user.id or shared_with(db, job_id, user) is not None:
|
||||
return row
|
||||
return None
|
||||
|
||||
|
||||
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob | None:
|
||||
"""A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's
|
||||
own in a folder it may read."""
|
||||
if caller.grants is None:
|
||||
return accessible_job(db, caller.user, job_id)
|
||||
row = own_job(db, caller.user, job_id)
|
||||
if row is None or not caller.may(row.folder_id, Right.READ):
|
||||
return None
|
||||
return row
|
||||
|
||||
|
||||
def asset_of(db: Session, job: DownloadJob | None) -> MediaAsset | None:
|
||||
"""The file under a job, if it has one yet."""
|
||||
return db.get(MediaAsset, job.asset_id) if job is not None and job.asset_id else None
|
||||
|
||||
|
||||
def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
|
||||
ids = {j.asset_id for j in jobs if j.asset_id}
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
|
||||
return {a.id: a for a in rows}
|
||||
|
||||
|
||||
def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
|
||||
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
|
||||
ids = [j.id for j in jobs]
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(
|
||||
select(LibraryState.job_id, LibraryState.position_seconds).where(
|
||||
owned_by(LibraryState.user_id, user), LibraryState.job_id.in_(ids)
|
||||
)
|
||||
).all()
|
||||
return {job_id: pos for job_id, pos in rows}
|
||||
|
||||
|
||||
def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]:
|
||||
"""Where each file came from (its source folder's key), for the list's Source column, the
|
||||
"back to its source folder" move and the download tray's site: a file in a source folder has its
|
||||
folder's key; one moved into an own folder (or someone else's, shared) is traced back by
|
||||
`folders.source_key_for`."""
|
||||
ids = {j.folder_id for j in jobs if j.folder_id is not None}
|
||||
by_id = (
|
||||
{f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
|
||||
if ids
|
||||
else {}
|
||||
)
|
||||
out: dict[int, str] = {}
|
||||
for j in jobs:
|
||||
folder = by_id.get(j.folder_id) if j.folder_id is not None else None
|
||||
if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key:
|
||||
out[j.id] = folder.source_key
|
||||
else:
|
||||
out[j.id] = folders.source_key_for(db, j)
|
||||
return out
|
||||
|
||||
|
||||
def queue(db: Session, user: User, recent: datetime) -> list[DownloadJob]:
|
||||
"""The download tray: what is waiting, running, paused or failed, and what finished (done or
|
||||
canceled) since `recent`. A done job is timed from when it became done, never from its last
|
||||
write: a rename, a move or the GC's expiry warning touches an old file's row, and the tray would
|
||||
announce it as just finished."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.where(
|
||||
owned_by(DownloadJob.user_id, user),
|
||||
DownloadJob.status.not_in(FINISHED)
|
||||
| ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent))
|
||||
| ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)),
|
||||
)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(MAX_LIST)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def statuses_by_source(db: Session, user: User, statuses: Sequence[str]) -> list[tuple[str, str]]:
|
||||
"""(source_ref, status) of each of `user`'s jobs in one of `statuses`."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob.source_ref, DownloadJob.status).where(
|
||||
owned_by(DownloadJob.user_id, user), DownloadJob.status.in_(tuple(statuses))
|
||||
)
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def library(
|
||||
db: Session,
|
||||
user: User,
|
||||
*,
|
||||
folder_id: int | None,
|
||||
listable: set[int] | None,
|
||||
shared: bool,
|
||||
term: str,
|
||||
sort: str,
|
||||
order: str,
|
||||
lang: str,
|
||||
) -> list[DownloadJob]:
|
||||
"""The caller's done files (optionally one folder's, or only in the folders a token may list),
|
||||
or the ones shared with them; searched by name and channel, sorted by when they were added, by
|
||||
name (in the reader's alphabet) or by size. A file is added to its owner's list when it is done,
|
||||
not when it was asked for (a download that failed for days and then went through is new); one
|
||||
shared with the caller, when it was shared."""
|
||||
stmt = (
|
||||
select(DownloadJob)
|
||||
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.where(DownloadJob.status == JobStatus.DONE)
|
||||
)
|
||||
if shared:
|
||||
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
|
||||
owned_by(DownloadShare.to_user_id, user)
|
||||
)
|
||||
else:
|
||||
stmt = stmt.where(owned_by(DownloadJob.user_id, user))
|
||||
if folder_id is not None:
|
||||
stmt = stmt.where(DownloadJob.folder_id == folder_id)
|
||||
elif listable is not None:
|
||||
stmt = stmt.where(DownloadJob.folder_id.in_(listable))
|
||||
name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
|
||||
if term:
|
||||
like = f"%{like_escape(term)}%"
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
name.ilike(like),
|
||||
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
|
||||
)
|
||||
)
|
||||
key = {
|
||||
"added": DownloadShare.created_at if shared else DownloadJob.ready_at,
|
||||
"name": collated(name, lang),
|
||||
"size": MediaAsset.size_bytes,
|
||||
}[sort]
|
||||
first = key.asc() if order == "asc" else key.desc()
|
||||
tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
|
||||
return list(db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars())
|
||||
|
||||
|
||||
def recent_jobs(db: Session) -> list[DownloadJob]:
|
||||
"""Everyone's newest jobs, for the admin's list."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(MAX_LIST)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
# --- shares --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def shared_with(db: Session, job_id: int, user: User | int) -> DownloadShare | None:
|
||||
"""The grant that hands this job to `user`, if there is one."""
|
||||
return db.execute(
|
||||
select(DownloadShare).where(
|
||||
DownloadShare.job_id == job_id, owned_by(DownloadShare.to_user_id, user)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]:
|
||||
rows = db.execute(
|
||||
select(DownloadShare.job_id, DownloadShare.created_at).where(
|
||||
owned_by(DownloadShare.to_user_id, user),
|
||||
DownloadShare.job_id.in_([j.id for j in jobs]),
|
||||
)
|
||||
).all()
|
||||
return {job_id: at for job_id, at in rows}
|
||||
|
||||
|
||||
# --- folders -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def own_folder(db: Session, user: User, folder_id: int) -> FileFolder | None:
|
||||
return owned_row(db, FileFolder, folder_id, user)
|
||||
|
||||
|
||||
def folders_by_id(db: Session, ids: set[int]) -> dict[int, FileFolder]:
|
||||
"""These folders, by id — for a reader that already knows which ones (a token's grants)."""
|
||||
if not ids:
|
||||
return {}
|
||||
return {f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
|
||||
|
||||
|
||||
def folders_with_totals(
|
||||
db: Session, user: User, listable: set[int] | None
|
||||
) -> list[tuple[FileFolder, int, int]]:
|
||||
"""The user's folders (only `listable` ones, when given) with how many done files each holds and
|
||||
their size. A source folder its files have all left stays (count 0)."""
|
||||
done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
|
||||
stmt = select(
|
||||
FileFolder,
|
||||
func.count(DownloadJob.id),
|
||||
func.coalesce(func.sum(MediaAsset.size_bytes), 0),
|
||||
).where(owned_by(FileFolder.user_id, user))
|
||||
if listable is not None:
|
||||
stmt = stmt.where(FileFolder.id.in_(listable))
|
||||
rows = db.execute(
|
||||
stmt.outerjoin(DownloadJob, done)
|
||||
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.group_by(FileFolder.id)
|
||||
.order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
|
||||
).tuples()
|
||||
return [(f, count, int(size or 0)) for f, count, size in rows]
|
||||
|
||||
|
||||
def folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
|
||||
count, size = db.execute(
|
||||
select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
|
||||
.select_from(DownloadJob)
|
||||
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
|
||||
).one()
|
||||
return count, int(size)
|
||||
|
||||
|
||||
def linked_folders(db: Session, folder_ids: list[int]) -> set[int]:
|
||||
"""Which of these folders a live public link opens (one not past its expiry)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = db.execute(
|
||||
select(DownloadLink.folder_id).where(
|
||||
DownloadLink.folder_id.in_(folder_ids),
|
||||
or_(DownloadLink.expires_at.is_(None), DownloadLink.expires_at > now),
|
||||
)
|
||||
).scalars()
|
||||
return {folder_id for folder_id in rows if folder_id is not None}
|
||||
|
||||
|
||||
# --- links ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def own_link(db: Session, user: User, link_id: int) -> DownloadLink | None:
|
||||
"""A link to one of the user's files or folders."""
|
||||
link = db.get(DownloadLink, link_id)
|
||||
if link is None:
|
||||
return None
|
||||
target = (
|
||||
db.get(DownloadJob, link.job_id)
|
||||
if link.job_id is not None
|
||||
else db.get(FileFolder, link.folder_id)
|
||||
)
|
||||
if target is None or target.user_id != user.id:
|
||||
return None
|
||||
return link
|
||||
|
||||
|
||||
def links_to(
|
||||
db: Session, *, job_id: int | None = None, folder_id: int | None = None
|
||||
) -> list[DownloadLink]:
|
||||
"""The links to one file or one folder, newest first."""
|
||||
target = (
|
||||
DownloadLink.job_id == job_id if job_id is not None else DownloadLink.folder_id == folder_id
|
||||
)
|
||||
return list(
|
||||
db.execute(select(DownloadLink).where(target).order_by(DownloadLink.id.desc())).scalars()
|
||||
)
|
||||
|
||||
|
||||
def link_folder(db: Session, link: DownloadLink) -> FileFolder | None:
|
||||
return db.get(FileFolder, link.folder_id) if link.folder_id is not None else None
|
||||
|
||||
|
||||
def folder_job(db: Session, folder: FileFolder, job_id: int) -> DownloadJob | None:
|
||||
"""One of the files a link to `folder` reaches (`links.in_folder`), while it is in it."""
|
||||
return db.execute(
|
||||
select(DownloadJob).where(DownloadJob.id == job_id, *links.in_folder(folder))
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def owner_suspended(db: Session, job: DownloadJob) -> bool:
|
||||
"""The job's owner is suspended (or gone): the worker leaves their queued jobs where they are."""
|
||||
owner = db.get(User, job.user_id)
|
||||
return owner is None or owner.is_suspended
|
||||
|
||||
|
||||
# --- profiles ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def profiles_for(db: Session, user: User) -> list[DownloadProfile]:
|
||||
"""The built-in format profiles, then the user's own."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadProfile)
|
||||
.where(DownloadProfile.is_builtin.is_(True) | owned_by(DownloadProfile.user_id, user))
|
||||
.order_by(
|
||||
DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def own_profile(db: Session, user: User, profile_id: int) -> DownloadProfile | None:
|
||||
"""One of the user's own profiles (a built-in is no one's to change)."""
|
||||
p = owned_row(db, DownloadProfile, profile_id, user)
|
||||
return None if p is None or p.is_builtin else p
|
||||
|
||||
|
||||
def usable_profile(db: Session, user: User, profile_id: int) -> DownloadProfile | None:
|
||||
"""A profile the user may enqueue with: a built-in, or their own."""
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or (not p.is_builtin and p.user_id != user.id):
|
||||
return None
|
||||
return p
|
||||
|
||||
|
||||
def default_profile(db: Session) -> DownloadProfile | None:
|
||||
"""The first built-in, for an enqueue that named no format."""
|
||||
return db.execute(
|
||||
select(DownloadProfile)
|
||||
.where(DownloadProfile.is_builtin.is_(True))
|
||||
.order_by(DownloadProfile.sort_order)
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
# --- storage and quota (admin) -------------------------------------------------------------
|
||||
|
||||
|
||||
def storage_totals(db: Session) -> tuple[int, int, int]:
|
||||
"""(files on disk, their bytes, every asset row). On disk is `quota.ON_DISK`: ready files, and
|
||||
uploads waiting for or failed in their conversion — the same disk the footprint and the GC cap
|
||||
count."""
|
||||
on_disk = db.execute(
|
||||
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).one()
|
||||
total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar_one()
|
||||
return on_disk[0], int(on_disk[1]), total_assets
|
||||
|
||||
|
||||
def done_files_per_user(db: Session) -> dict[int, int]:
|
||||
"""How many finished files each user's Files lists."""
|
||||
return dict(
|
||||
db.execute(
|
||||
select(DownloadJob.user_id, func.count())
|
||||
.where(DownloadJob.status == JobStatus.DONE)
|
||||
.group_by(DownloadJob.user_id)
|
||||
)
|
||||
.tuples()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def custom_quota(db: Session, user_id: int) -> DownloadQuota | None:
|
||||
"""The admin's own limits for this user, if any were set."""
|
||||
return db.get(DownloadQuota, user_id)
|
||||
@@ -26,7 +26,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.downloads.status import JobKind, JobStatus
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.userscope import owned_by
|
||||
|
||||
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
||||
# (error, canceled) don't count.
|
||||
@@ -85,7 +86,7 @@ def footprint(db: Session, user_id: int) -> int:
|
||||
held_assets = (
|
||||
select(DownloadJob.asset_id)
|
||||
.where(
|
||||
DownloadJob.user_id == user_id,
|
||||
owned_by(DownloadJob.user_id, user_id),
|
||||
or_(
|
||||
DownloadJob.status.in_(_HOLDING),
|
||||
and_(DownloadJob.job_kind == JobKind.UPLOAD, DownloadJob.status == JobStatus.ERROR),
|
||||
@@ -109,7 +110,7 @@ def _count(db: Session, user_id: int, states) -> int:
|
||||
db.execute(
|
||||
select(func.count())
|
||||
.select_from(DownloadJob)
|
||||
.where(DownloadJob.user_id == user_id, DownloadJob.status.in_(states))
|
||||
.where(owned_by(DownloadJob.user_id, user_id), DownloadJob.status.in_(states))
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
@@ -201,3 +202,46 @@ def usage(db: Session, user_id: int) -> dict:
|
||||
"max_jobs": lim.max_jobs,
|
||||
"unlimited": lim.unlimited,
|
||||
}
|
||||
|
||||
|
||||
def set_custom(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
*,
|
||||
max_bytes: int | None = None,
|
||||
max_concurrent: int | None = None,
|
||||
max_jobs: int | None = None,
|
||||
unlimited: bool | None = None,
|
||||
) -> None:
|
||||
"""The admin's own limits for a user: starts from what applies now, then each value given (a
|
||||
size floored at 0, a count at 1). The handler commits."""
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is None:
|
||||
cur = resolve(db, user_id)
|
||||
row = DownloadQuota(
|
||||
user_id=user_id,
|
||||
max_bytes=cur.max_bytes,
|
||||
max_concurrent=cur.max_concurrent,
|
||||
max_jobs=cur.max_jobs,
|
||||
unlimited=cur.unlimited,
|
||||
)
|
||||
db.add(row)
|
||||
if max_bytes is not None:
|
||||
row.max_bytes = max(0, max_bytes)
|
||||
if max_concurrent is not None:
|
||||
row.max_concurrent = max(1, max_concurrent)
|
||||
if max_jobs is not None:
|
||||
row.max_jobs = max(1, max_jobs)
|
||||
if unlimited is not None:
|
||||
row.unlimited = unlimited
|
||||
db.flush()
|
||||
|
||||
|
||||
def reset_custom(db: Session, user_id: int) -> bool:
|
||||
"""Back to the defaults; False when the user had no limits of their own."""
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is None:
|
||||
return False
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
return True
|
||||
|
||||
@@ -23,7 +23,7 @@ from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats
|
||||
from app.downloads.status import JobKind
|
||||
from app.models import DownloadJob
|
||||
from app.downloads.models import DownloadJob
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
"""Enqueue + cache resolution for the download center (the per-user layer over MediaAsset).
|
||||
"""Capture's writes: enqueue + cache resolution, the job transitions, and what the library's
|
||||
dialogs change (profiles, resume points, shares, a file's display fields).
|
||||
|
||||
`enqueue` is the single entry point used by the API: it resolves (or creates) the shared
|
||||
MediaAsset for the requested source+format, links a per-user DownloadJob to it, and — on a
|
||||
cache hit (asset already ready) — marks the job done instantly with no re-download. The worker
|
||||
handles everything else. State transitions used by the routes (pause/resume/cancel/delete) and
|
||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||
handles everything else. Every function here flushes and leaves the commit to its caller (the
|
||||
handler); the folder, link and quota writes are `folders`, `links` and `quota`.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import delete as delete_rows
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -21,17 +24,17 @@ from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import folders, formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobKind, JobStatus
|
||||
from app.models import (
|
||||
DISPLAY_TEXT_MAX,
|
||||
Channel,
|
||||
from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, User, Video
|
||||
from app.downloads.models import (
|
||||
DownloadJob,
|
||||
DownloadProfile,
|
||||
DownloadShare,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
PlexItem,
|
||||
PlexShow,
|
||||
Video,
|
||||
)
|
||||
from app.plex import lookups as plex_lookups
|
||||
from app.plex import paths as plex_paths
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
|
||||
class EditError(Exception):
|
||||
@@ -230,8 +233,7 @@ def enqueue(
|
||||
folders.stamp_done(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
db.flush()
|
||||
return job
|
||||
|
||||
|
||||
@@ -310,8 +312,7 @@ def enqueue_edit(
|
||||
folders.stamp_done(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
db.flush()
|
||||
return job
|
||||
|
||||
|
||||
@@ -399,8 +400,7 @@ def enqueue_clip(
|
||||
folders.stamp_done(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
db.flush()
|
||||
return job
|
||||
|
||||
|
||||
@@ -508,8 +508,8 @@ def release_asset(db: Session, job: DownloadJob) -> None:
|
||||
# --- Job state transitions ---------------------------------------------------------------------
|
||||
# The single home for the pause/resume/cancel/delete/redownload transitions + their asset
|
||||
# bookkeeping (release, failed-asset reset, fresh-asset rebuild), so the routes are thin callers and
|
||||
# the worker/GC could share the same definitions (B-sub-8). Each commits its own transition; the
|
||||
# route keeps only auth + HTTP mapping + serialization.
|
||||
# the worker/GC could share the same definitions (B-sub-8). Each flushes its transition and the caller
|
||||
# commits it; the route keeps only auth + HTTP mapping + serialization.
|
||||
|
||||
|
||||
def pause(db: Session, job: DownloadJob) -> None:
|
||||
@@ -517,7 +517,7 @@ def pause(db: Session, job: DownloadJob) -> None:
|
||||
in any other state."""
|
||||
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING):
|
||||
job.status = JobStatus.PAUSED
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
def resume(db: Session, job: DownloadJob) -> None:
|
||||
@@ -541,7 +541,7 @@ def resume(db: Session, job: DownloadJob) -> None:
|
||||
asset.status = "pending"
|
||||
asset.error = None
|
||||
job.status = JobStatus.QUEUED
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
def _follow_the_cookie(db: Session, job: DownloadJob, asset: MediaAsset) -> MediaAsset:
|
||||
@@ -576,14 +576,14 @@ def cancel(db: Session, job: DownloadJob) -> None:
|
||||
if job.status not in (JobStatus.DONE, JobStatus.CANCELED):
|
||||
release_asset(db, job) # release while the job still counts as holding
|
||||
job.status = JobStatus.CANCELED
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
def delete(db: Session, job: DownloadJob) -> None:
|
||||
"""Delete a job, releasing its asset hold first (which frees the file iff this was the last holder)."""
|
||||
release_asset(db, job)
|
||||
db.delete(job)
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
def redownload(db: Session, job: DownloadJob) -> None:
|
||||
@@ -632,4 +632,181 @@ def redownload(db: Session, job: DownloadJob) -> None:
|
||||
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
|
||||
job.expires_at = None # stamped again when the rebuild is done
|
||||
job.expiry_warned = False
|
||||
db.commit()
|
||||
db.flush()
|
||||
|
||||
|
||||
# --- the library's writes ------------------------------------------------------------------
|
||||
# What the HTTP layer changes for a user: each flushes, the handler commits.
|
||||
|
||||
|
||||
def lock_own_job(db: Session, user: User, job_id: int) -> DownloadJob | None:
|
||||
"""One of the user's jobs, re-read under a row lock, for a transition that moves its asset's ref
|
||||
count (resume, cancel, delete, redownload) or writes the status those decide on (pause). Two
|
||||
overlapping requests for one job run one after the other and the second decides on the first's
|
||||
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||
just released back into a holder; either can free a file another job still holds. None: not
|
||||
theirs, or gone.
|
||||
|
||||
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||
(`finalize_if_running`), so the two can never deadlock."""
|
||||
while True:
|
||||
owned = owned_row(db, DownloadJob, job_id, user)
|
||||
if owned is None:
|
||||
return None
|
||||
asset_id = owned.asset_id
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
return None
|
||||
if job.asset_id == asset_id:
|
||||
return job
|
||||
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||
|
||||
|
||||
def hold_for_share(db: Session, user: User, job_id: int) -> DownloadJob | None:
|
||||
"""One of the user's files, held (FOR SHARE) for the rest of the transaction: a delete of it at
|
||||
the same moment (the user's, or the retention sweep's) waits and takes the new link with it,
|
||||
rather than failing the link's foreign key — or, done first, leaves None."""
|
||||
if owned_row(db, DownloadJob, job_id, user) is None:
|
||||
return None
|
||||
return db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update(read=True)
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def create_profile(db: Session, user: User, name: str, spec: dict) -> DownloadProfile:
|
||||
"""A format profile of the user's own, after their others."""
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
|
||||
owned_by(DownloadProfile.user_id, user)
|
||||
)
|
||||
).scalar_one()
|
||||
p = DownloadProfile(
|
||||
user_id=user.id, name=name[:80], spec=spec, is_builtin=False, sort_order=maxpos + 1
|
||||
)
|
||||
db.add(p)
|
||||
db.flush()
|
||||
return p
|
||||
|
||||
|
||||
def update_profile(profile: DownloadProfile, *, name: str | None, spec: dict | None) -> None:
|
||||
"""Rename a profile and/or replace its spec; None leaves that one as it is."""
|
||||
if name is not None:
|
||||
profile.name = name[:80]
|
||||
if spec is not None:
|
||||
profile.spec = spec
|
||||
|
||||
|
||||
def delete_profile(db: Session, profile: DownloadProfile) -> None:
|
||||
db.delete(profile)
|
||||
db.flush()
|
||||
|
||||
|
||||
def set_position(db: Session, user: User, job_id: int, position: int) -> None:
|
||||
"""The library player's resume point in a file the user can play; 0 clears it.
|
||||
|
||||
A clear is a statement, not a loaded row's delete: the periodic checkpoint and the pagehide beacon
|
||||
of one stop may both clear it, and the second then deletes nothing rather than failing. A set is
|
||||
an upsert, one row per (user, job) even when two checkpoints race, not a read-then-insert that
|
||||
the second would fail on the unique key."""
|
||||
if not position:
|
||||
db.execute(
|
||||
delete_rows(LibraryState).where(
|
||||
owned_by(LibraryState.user_id, user), LibraryState.job_id == job_id
|
||||
)
|
||||
)
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
db.execute(
|
||||
pg_insert(LibraryState)
|
||||
.values(user_id=user.id, job_id=job_id, position_seconds=position, progress_updated_at=now)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_library_user_job",
|
||||
set_={"position_seconds": position, "progress_updated_at": now},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def share(db: Session, job: DownloadJob, sender: User, recipient: User) -> bool:
|
||||
"""Hand a file to another user, with a notice for them; False when it was already theirs."""
|
||||
from app.notifications import create_notification
|
||||
|
||||
exists = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job.id, owned_by(DownloadShare.to_user_id, recipient)
|
||||
)
|
||||
).first()
|
||||
if exists is not None:
|
||||
return False
|
||||
db.add(DownloadShare(job_id=job.id, from_user_id=sender.id, to_user_id=recipient.id))
|
||||
create_notification(
|
||||
db,
|
||||
user_id=recipient.id,
|
||||
type="download_shared",
|
||||
title=(job.display_name or "A download") + " was shared with you",
|
||||
# The client names the notice in the reader's language from `data`; the title is the
|
||||
# English fallback only.
|
||||
data={
|
||||
"kind": "download_shared",
|
||||
"from": sender.email,
|
||||
"job_id": job.id,
|
||||
"title": job.display_name,
|
||||
},
|
||||
commit=False,
|
||||
)
|
||||
db.flush()
|
||||
return True
|
||||
|
||||
|
||||
def unshare(db: Session, job_id: int, recipient: User) -> bool:
|
||||
"""Take a file back from one user (its owner's act, or the recipient's own dismissal); False
|
||||
when it was not theirs."""
|
||||
row = db.execute(
|
||||
select(DownloadShare).where(
|
||||
DownloadShare.job_id == job_id, owned_by(DownloadShare.to_user_id, recipient)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return False
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
return True
|
||||
|
||||
|
||||
# The display fields a user may set on their file; the file on disk keeps its own name.
|
||||
DISPLAY_FIELDS = frozenset(
|
||||
{"display_name", "display_uploader", "display_uploader_url", "extra_links"}
|
||||
)
|
||||
|
||||
|
||||
def set_display(job: DownloadJob, **fields: object) -> None:
|
||||
"""Set a file's display fields (each already cleaned by the caller); a field not given is left."""
|
||||
unknown = set(fields) - DISPLAY_FIELDS
|
||||
if unknown:
|
||||
raise ValueError(f"not a display field: {sorted(unknown)}")
|
||||
for name, value in fields.items():
|
||||
setattr(job, name, value)
|
||||
|
||||
|
||||
def touch(asset: MediaAsset) -> None:
|
||||
"""The file was just served: the cache's idle clock starts again."""
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def store_sha256(db: Session, asset_id: int, digest: str) -> None:
|
||||
"""Keep a file's hash once it is known — only where none is, so a concurrent hash of the same
|
||||
file never overwrites the first."""
|
||||
db.execute(
|
||||
update(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
|
||||
.values(sha256=digest)
|
||||
)
|
||||
|
||||
@@ -37,7 +37,8 @@ from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
|
||||
from app.downloads.status import JobKind, JobStatus
|
||||
from app.models import DownloadJob, MediaAsset, display_text
|
||||
from app.models import display_text
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def _title_from_filename(filename: str) -> str:
|
||||
|
||||
+114
-21
@@ -1,10 +1,11 @@
|
||||
"""Feed read-model: the pure query-building and row-serialization layer behind the feed.
|
||||
"""Feed read-model: Reader's query-building, paging and row serialization behind the feed.
|
||||
|
||||
Extracted from `routes/feed.py` so the read-model has one home instead of living inside the
|
||||
router module — the feed route handlers and the playlist-detail query share these helpers (the
|
||||
projection, the filter builder, and keyset paging). Nothing here defines a route or reaches for
|
||||
request state; it's all `(db, user, params) -> query`, and a malformed cursor raises a plain
|
||||
`ValueError` for the route handler to translate (the tsquery sanitizer lives in `app.utils`).
|
||||
One home for the read-model rather than the router module: the feed routes, the playlist-detail
|
||||
query and Home share these helpers (the projection, the filter builder, and keyset paging), and the
|
||||
feed's routes only validate and answer — a page, its counts and its facets are built here. Nothing
|
||||
here defines a route or reaches for request state; it's all `(db, user, params) -> query`, and a
|
||||
malformed cursor raises `BadCursor` for the route handler to translate (the tsquery sanitizer lives
|
||||
in `app.utils`).
|
||||
"""
|
||||
|
||||
import base64
|
||||
@@ -20,18 +21,15 @@ from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
ChannelTag,
|
||||
ExploredChannel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SearchFind,
|
||||
Subscription,
|
||||
Tag,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, Tag, VideoState
|
||||
from app.utils import iso, like_escape, to_tsquery_str
|
||||
from app.userscope import owned_by
|
||||
|
||||
|
||||
def _channel_url(channel_id: str, handle: str | None) -> str:
|
||||
@@ -48,7 +46,7 @@ def _saved_expr(user: User):
|
||||
.select_from(PlaylistItem)
|
||||
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
|
||||
.where(
|
||||
Playlist.user_id == user.id,
|
||||
owned_by(Playlist.user_id, user.id),
|
||||
Playlist.kind == "watch_later",
|
||||
PlaylistItem.video_id == Video.id,
|
||||
)
|
||||
@@ -103,7 +101,7 @@ def serialize(row) -> dict:
|
||||
|
||||
def _tag_visible(user: User):
|
||||
"""A ChannelTag row this user may see: a shared system tag (user_id NULL) or their own."""
|
||||
return or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||
return or_(ChannelTag.user_id.is_(None), owned_by(ChannelTag.user_id, user.id))
|
||||
|
||||
|
||||
# The pseudo-category of a selected tag id no Tag row answers (a saved view or the feed state outlives
|
||||
@@ -122,7 +120,7 @@ def tags_by_category(db: Session, user: User, tag_ids: list[int]) -> dict[str, l
|
||||
if not tag_ids:
|
||||
return {} # skip the pointless `WHERE id IN ()` round-trip on the no-tag-filter hot path
|
||||
out: dict[str, list[int]] = {}
|
||||
visible = or_(Tag.user_id.is_(None), Tag.user_id == user.id)
|
||||
visible = or_(Tag.user_id.is_(None), owned_by(Tag.user_id, user.id))
|
||||
for tid, cat in db.execute(
|
||||
select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids), visible)
|
||||
).all():
|
||||
@@ -142,7 +140,7 @@ def not_blocked_cond(user: User):
|
||||
position — and the one query that copied only PART of this predicate set is exactly how that
|
||||
video came back as the landing page's suggestion after the user had said "never again"."""
|
||||
return Video.channel_id.notin_(
|
||||
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
|
||||
select(BlockedChannel.channel_id).where(owned_by(BlockedChannel.user_id, user.id))
|
||||
)
|
||||
|
||||
|
||||
@@ -187,7 +185,7 @@ def filtered_query(
|
||||
Subscription,
|
||||
and_(
|
||||
Subscription.channel_id == Video.channel_id,
|
||||
Subscription.user_id == user.id,
|
||||
owned_by(Subscription.user_id, user.id),
|
||||
),
|
||||
)
|
||||
else:
|
||||
@@ -199,12 +197,12 @@ def filtered_query(
|
||||
Subscription,
|
||||
and_(
|
||||
Subscription.channel_id == Video.channel_id,
|
||||
Subscription.user_id == user.id,
|
||||
owned_by(Subscription.user_id, user.id),
|
||||
Subscription.hidden.is_(False),
|
||||
),
|
||||
).outerjoin(
|
||||
SearchFind,
|
||||
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
|
||||
and_(SearchFind.video_id == Video.id, owned_by(SearchFind.user_id, user.id)),
|
||||
)
|
||||
|
||||
# The state-gated views (watched/hidden/in_progress) only ever return videos that HAVE a
|
||||
@@ -227,8 +225,8 @@ def filtered_query(
|
||||
# catalog. Applied in BOTH scopes; the channel page reaches its ephemeral videos by
|
||||
# asking for scope=all + channel_ids (the channel is in this accessible set).
|
||||
accessible_explore = (
|
||||
select(Subscription.channel_id).where(Subscription.user_id == user.id)
|
||||
).union(select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id))
|
||||
select(Subscription.channel_id).where(owned_by(Subscription.user_id, user.id))
|
||||
).union(select(ExploredChannel.channel_id).where(owned_by(ExploredChannel.user_id, user.id)))
|
||||
query = query.where(
|
||||
or_(
|
||||
Video.via_explore.is_(False),
|
||||
@@ -346,7 +344,7 @@ def filtered_query(
|
||||
if untagged:
|
||||
query = query.where(
|
||||
Video.channel_id.notin_(
|
||||
select(ChannelTag.channel_id).where(ChannelTag.user_id == user.id)
|
||||
select(ChannelTag.channel_id).where(owned_by(ChannelTag.user_id, user.id))
|
||||
)
|
||||
)
|
||||
|
||||
@@ -696,3 +694,98 @@ def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str, datetime |
|
||||
# Framework-free: the route handler translates this into an HTTP 400 so the
|
||||
# read-model stays web-layer-agnostic (see routes/feed.get_feed).
|
||||
raise ValueError("Invalid feed cursor") from exc
|
||||
|
||||
|
||||
class BadCursor(ValueError):
|
||||
"""A `cursor` this module did not write (or one from another sort): the caller's bad input."""
|
||||
|
||||
|
||||
def feed_page(
|
||||
db: Session,
|
||||
user: User,
|
||||
params: dict,
|
||||
*,
|
||||
sort: str,
|
||||
seed: int,
|
||||
limit: int,
|
||||
cursor: str | None,
|
||||
lang: str,
|
||||
) -> dict:
|
||||
"""One page of the feed, and the cursor for the next. A malformed cursor raises `BadCursor`."""
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys(sort, seed, rank_expr, lang)
|
||||
|
||||
vals, cursor_id, window = None, None, None
|
||||
if cursor:
|
||||
try:
|
||||
vals, cursor_id, window = decode_cursor(keys, cursor)
|
||||
except ValueError as e:
|
||||
raise BadCursor(str(e)) from e
|
||||
ranked = sort == "relevance" and rank_expr is not None
|
||||
if ranked and not cursor:
|
||||
window = rank_window_start(db, query)
|
||||
if ranked and window is not None:
|
||||
keys = sort_keys(sort, seed, windowed_rank(rank_expr, window), lang)
|
||||
|
||||
# Expose each key column on the row so we can build the next cursor from the last item.
|
||||
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
||||
if vals is not None:
|
||||
for k, v in zip(keys, vals, strict=True):
|
||||
k["value"] = v
|
||||
if ranked and window is not None:
|
||||
rows = windowed_relevance_rows(db, query, keys, window, cursor_id, limit + 1)
|
||||
else:
|
||||
query = query.order_by(*order_by(keys))
|
||||
if cursor_id is not None:
|
||||
query = query.where(keyset_predicate(keys, cursor_id))
|
||||
rows = db.execute(query.limit(limit + 1)).all()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
return {
|
||||
"items": [serialize(r) for r in page],
|
||||
"next_cursor": (
|
||||
encode_cursor(keys, page[-1], window if ranked else None)
|
||||
if (has_more and page)
|
||||
else None
|
||||
),
|
||||
"has_more": has_more,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
|
||||
def feed_count(db: Session, user: User, **params) -> int:
|
||||
"""How many videos the filters match — the reference oracle `feed_summary`'s count is tested
|
||||
against."""
|
||||
query, _rank = filtered_query(db, user, **params)
|
||||
return db.scalar(select(func.count()).select_from(query.subquery())) or 0
|
||||
|
||||
|
||||
def facet_counts(db: Session, user: User, **params) -> dict[int, int]:
|
||||
"""Per-tag channel counts for the current filter context, one pass per category — the reference
|
||||
oracle `feed_summary`'s facets are tested against.
|
||||
|
||||
Each category is counted with every other filter applied but its own selections ignored
|
||||
(drill-down faceting: selecting one topic doesn't zero out the rest). A conjunctive (AND)
|
||||
category keeps them applied, so each remaining chip narrows to channels that ALSO have every
|
||||
selected tag, and tags that can't co-occur drop to zero."""
|
||||
visible = _tag_visible(user)
|
||||
counts: dict[int, int] = {}
|
||||
for category in ("language", "topic", "other"):
|
||||
conjunctive = params.get("tag_modes", {}).get(category) == "and"
|
||||
base, _rank = filtered_query(
|
||||
db,
|
||||
user,
|
||||
**{**params, "exclude_tag_category": None if conjunctive else category},
|
||||
)
|
||||
channels = base.with_only_columns(Video.channel_id).distinct().subquery()
|
||||
rows = db.execute(
|
||||
select(ChannelTag.tag_id, func.count(func.distinct(ChannelTag.channel_id)))
|
||||
.select_from(channels)
|
||||
.join(ChannelTag, ChannelTag.channel_id == channels.c.channel_id)
|
||||
.join(Tag, and_(Tag.id == ChannelTag.tag_id, Tag.category == category))
|
||||
.where(visible)
|
||||
.group_by(ChannelTag.tag_id)
|
||||
).all()
|
||||
for tag_id, count in rows:
|
||||
counts[tag_id] = count
|
||||
return counts
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
"""The durable history of background runs — the `job_runs` table.
|
||||
|
||||
A run is written twice: a `running` row when it starts, its outcome when it ends. Each write is its
|
||||
OWN short session, committed at once, for the reason `quota.record_usage` gives: the run's own work
|
||||
may roll back (its `except: db.rollback()`), and the history of what happened must not go with it —
|
||||
nor may this bookkeeping commit the job's half-done session.
|
||||
|
||||
Only the API process runs jobs (the worker never starts the scheduler), and one API process serves a
|
||||
database (`.env.example`, the Scheduler section: the single-run guard is in its memory too), so at its
|
||||
startup nothing can still be running: `interrupt_leftovers` closes what a stopped process left open.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import delete, func, select, union_all, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.models import JobRun, QuotaEvent
|
||||
from app.runstatus import RunStatus, RunTrigger
|
||||
|
||||
|
||||
def start(job: str, trigger: RunTrigger, actor_id: int | None, target: str | None = None) -> int:
|
||||
"""Open a run's row; returns its id."""
|
||||
with SessionLocal() as s:
|
||||
row = JobRun(
|
||||
job=job,
|
||||
trigger=trigger,
|
||||
actor_id=actor_id,
|
||||
target=target,
|
||||
status=RunStatus.RUNNING,
|
||||
started_at=datetime.now(timezone.utc),
|
||||
)
|
||||
s.add(row)
|
||||
s.commit()
|
||||
return row.id
|
||||
|
||||
|
||||
def finish(
|
||||
run_id: int,
|
||||
status: RunStatus,
|
||||
*,
|
||||
summary: str | None = None,
|
||||
result: dict | None = None,
|
||||
error: str | None = None,
|
||||
failure: dict | None = None,
|
||||
) -> None:
|
||||
"""Close a run with its outcome and what it spent. Conditional on the row still running: a row
|
||||
the startup sweep already called interrupted is not rewritten by a thread that outlived it."""
|
||||
with SessionLocal() as s:
|
||||
units = s.scalar(
|
||||
select(func.coalesce(func.sum(QuotaEvent.units), 0)).where(
|
||||
QuotaEvent.job_run_id == run_id
|
||||
)
|
||||
)
|
||||
s.execute(
|
||||
update(JobRun)
|
||||
.where(JobRun.id == run_id, JobRun.status == RunStatus.RUNNING)
|
||||
.values(
|
||||
status=status,
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
summary=summary,
|
||||
result=result,
|
||||
error=error,
|
||||
failure=failure,
|
||||
quota_units=units or 0,
|
||||
)
|
||||
)
|
||||
s.commit()
|
||||
|
||||
|
||||
def interrupt_leftovers() -> int:
|
||||
"""Every row a previous process left `running` → `interrupted`, with what it spent until then
|
||||
(its quota rows outlive it, as `finish` sums them). Called once at startup, before the scheduler
|
||||
can open a new run. Returns how many."""
|
||||
spent = (
|
||||
select(func.coalesce(func.sum(QuotaEvent.units), 0))
|
||||
.where(QuotaEvent.job_run_id == JobRun.id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
with SessionLocal() as s:
|
||||
res = s.execute(
|
||||
update(JobRun)
|
||||
.where(JobRun.status == RunStatus.RUNNING)
|
||||
.values(
|
||||
status=RunStatus.INTERRUPTED,
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
quota_units=spent,
|
||||
)
|
||||
)
|
||||
s.commit()
|
||||
return res.rowcount if isinstance(res, CursorResult) else 0
|
||||
|
||||
|
||||
def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[JobRun]]:
|
||||
"""Each of `jobs`' last `per_job` runs, newest first, by job id — one query, each job's part a
|
||||
short walk of `ix_job_runs_job_started`, so a poll costs the same at any history length (a window
|
||||
over the whole table grew with every run kept)."""
|
||||
if not jobs:
|
||||
return {}
|
||||
newest = union_all(
|
||||
*(
|
||||
select(JobRun.id)
|
||||
.where(JobRun.job == job)
|
||||
.order_by(JobRun.started_at.desc(), JobRun.id.desc())
|
||||
.limit(per_job)
|
||||
for job in jobs
|
||||
)
|
||||
).subquery()
|
||||
rows = db.execute(
|
||||
select(JobRun)
|
||||
.where(JobRun.id.in_(select(newest.c.id)))
|
||||
.order_by(JobRun.job, JobRun.started_at.desc(), JobRun.id.desc())
|
||||
).scalars()
|
||||
out: dict[str, list[JobRun]] = {}
|
||||
for r in rows:
|
||||
out.setdefault(r.job, []).append(r)
|
||||
return out
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Delete finished runs older than `retention_days` (0 = keep forever). A run still going is
|
||||
never reaped. The caller commits."""
|
||||
if retention_days <= 0:
|
||||
return 0
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
|
||||
res = db.execute(
|
||||
delete(JobRun).where(JobRun.started_at < cutoff, JobRun.status != RunStatus.RUNNING)
|
||||
)
|
||||
return res.rowcount if isinstance(res, CursorResult) else 0
|
||||
+15
-2
@@ -10,8 +10,8 @@ the logs were both unjoinable and leaky. `configure_logging()` replaces all four
|
||||
Event shape — the OpenTelemetry log data model, without the SDK; one JSON object per line:
|
||||
ts · level · sev · source · logger · event · msg · attrs
|
||||
plus whatever correlation the context carries (`request_id`, `user_id`, `debug_session_id`,
|
||||
`job_id`) and the app version. That shape is what the admin log console filters on and what a Gitea
|
||||
issue attachment will carry, so it is deliberately flat and stable.
|
||||
`job_id`, `run_id`) and the app version. That shape is what the admin log console filters on and
|
||||
what a Gitea issue attachment will carry, so it is deliberately flat and stable.
|
||||
|
||||
Call sites stay ordinary stdlib logging:
|
||||
|
||||
@@ -81,15 +81,28 @@ _debug_session: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"log_debug_session", default=None
|
||||
)
|
||||
_job_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_job_id", default=None)
|
||||
# The background run (app/jobruns.py) this context is doing; quota's spend is tagged with it too.
|
||||
_run_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_run_id", default=None)
|
||||
|
||||
_CONTEXT_VARS = {
|
||||
"request_id": _request_id,
|
||||
"user_id": _user_id,
|
||||
"debug_session_id": _debug_session,
|
||||
"job_id": _job_id,
|
||||
"run_id": _run_id,
|
||||
}
|
||||
|
||||
|
||||
def current_request_id() -> str | None:
|
||||
"""The HTTP request this context is serving, if any — the request middleware sets it."""
|
||||
return _request_id.get()
|
||||
|
||||
|
||||
def current_run_id() -> int | None:
|
||||
"""The background run this context is doing, if any — `bind(run_id=…)` sets it."""
|
||||
return _run_id.get()
|
||||
|
||||
|
||||
class _RequestScope:
|
||||
"""What the auth dependency learns about a request, for every line logged after it.
|
||||
|
||||
|
||||
@@ -362,6 +362,7 @@ def serialize(row: LogEvent) -> dict:
|
||||
"request_id": row.request_id,
|
||||
"user_id": row.user_id,
|
||||
"job_id": row.job_id,
|
||||
"run_id": row.run_id,
|
||||
"debug_session_id": row.debug_session_id,
|
||||
"version": row.version,
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ from app.routes import ( # noqa: E402 — logging is configured first
|
||||
plex as plex_routes,
|
||||
public as public_routes,
|
||||
quota,
|
||||
runs as runs_routes,
|
||||
saved_views,
|
||||
scheduler as scheduler_routes,
|
||||
search as search_routes,
|
||||
@@ -345,6 +346,7 @@ app.include_router(config_routes.router)
|
||||
app.include_router(scheduler_routes.router)
|
||||
app.include_router(audit_routes.router)
|
||||
app.include_router(admin_logs_routes.router)
|
||||
app.include_router(runs_routes.router)
|
||||
app.include_router(quota.router)
|
||||
app.include_router(version.router)
|
||||
app.include_router(setup_routes.router)
|
||||
|
||||
+56
-552
@@ -1,4 +1,4 @@
|
||||
from datetime import date, datetime, timezone
|
||||
from datetime import date, datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
@@ -18,12 +18,12 @@ from sqlalchemy import (
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
|
||||
from app.db import Base
|
||||
from app.runstatus import RunStatus, RunTrigger, sql_in
|
||||
from app.utils import canonical_email
|
||||
|
||||
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
||||
@@ -406,85 +406,6 @@ class Video(Base, TimestampMixin):
|
||||
channel: Mapped["Channel"] = relationship(back_populates="videos")
|
||||
|
||||
|
||||
class Tag(Base, TimestampMixin):
|
||||
"""A label. System tags (user_id NULL) are computed automatically and shared by all
|
||||
users; user tags (user_id set) are personal. category groups them in the UI."""
|
||||
|
||||
__tablename__ = "tags"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(64))
|
||||
color: Mapped[str | None] = mapped_column(String(16))
|
||||
# language | topic | other
|
||||
category: Mapped[str] = mapped_column(String(16), default="other", server_default="other")
|
||||
|
||||
|
||||
class ChannelTag(Base):
|
||||
"""Links a tag to a channel. source='auto' rows are managed by the auto-tagger and
|
||||
regenerated freely; source='user' rows are never touched by the system."""
|
||||
|
||||
__tablename__ = "channel_tags"
|
||||
__table_args__ = (UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
tag_id: Mapped[int] = mapped_column(ForeignKey("tags.id", ondelete="CASCADE"), index=True)
|
||||
# NULL = system/auto link visible to everyone; set = a specific user's link.
|
||||
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
|
||||
confidence: Mapped[float | None] = mapped_column(Float)
|
||||
|
||||
|
||||
class VideoState(Base, UpdatedAtMixin):
|
||||
"""Per-user watch state for a video. Rows exist only for non-default states; the
|
||||
absence of a row means 'new/unseen'."""
|
||||
|
||||
__tablename__ = "video_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_video"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||
# new | watched | hidden ("saved" is now Watch-later playlist membership, not a status)
|
||||
status: Mapped[str] = mapped_column(String(12), default="new", server_default="new")
|
||||
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Resume position (seconds into the video) for the in-app player. Orthogonal to
|
||||
# status: a partially-watched video keeps status "new" but a non-zero position, so
|
||||
# it can render a progress bar and match the "in progress" feed filter. 0 = not
|
||||
# started / finished (trivially-early and near-finished positions are not stored).
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(
|
||||
Integer, default=0, server_default="0", nullable=False
|
||||
)
|
||||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class LibraryState(Base):
|
||||
"""Where a user stopped in a library item (their own download, or one shared with them) — the
|
||||
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
|
||||
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
|
||||
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
|
||||
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
|
||||
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
|
||||
|
||||
__tablename__ = "library_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class SearchFind(Base, TimestampMixin):
|
||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||
|
||||
@@ -563,14 +484,20 @@ class QuotaEvent(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), index=True
|
||||
)
|
||||
# The background run that spent it (app/jobruns.py); NULL = a request's own spend. SET NULL: a
|
||||
# reaped run leaves the spend in the budget's history.
|
||||
job_run_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("job_runs.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
|
||||
|
||||
class AuditLog(Base):
|
||||
"""Append-only admin/scheduler audit trail: who did what, when, and (where it applies)
|
||||
the before->after values. `actor_id` is the acting admin; NULL means the scheduler /
|
||||
background did it (SET NULL on user delete keeps the trail). Logged at ACTION / job-run
|
||||
granularity — never per-row — so a scheduler run that touches thousands of videos is ONE
|
||||
row summarising the outcome. `before`/`after` are small JSON snapshots (secrets excluded)."""
|
||||
"""Append-only admin audit trail: who did what, when, and (where it applies) the before->after
|
||||
values. `actor_id` is the acting admin; NULL means the background did it (SET NULL on user
|
||||
delete keeps the trail). Logged per ACTION, never per row; a background run is not an action
|
||||
and has its own history (`JobRun`). `before`/`after` are small JSON snapshots (secrets
|
||||
excluded). `request_id` is the request that made the change, so its log lines are one click
|
||||
away (NULL for rows written outside a request, and for rows older than the column)."""
|
||||
|
||||
__tablename__ = "audit_log"
|
||||
|
||||
@@ -587,6 +514,7 @@ class AuditLog(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), index=True
|
||||
)
|
||||
request_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
|
||||
|
||||
class LogEvent(Base):
|
||||
@@ -609,6 +537,8 @@ class LogEvent(Base):
|
||||
"debug_session_id",
|
||||
postgresql_where="debug_session_id IS NOT NULL",
|
||||
),
|
||||
# A run's lines are `WHERE run_id = …`; request lines carry none.
|
||||
Index("ix_log_events_run", "run_id", postgresql_where="run_id IS NOT NULL"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
@@ -624,7 +554,11 @@ class LogEvent(Base):
|
||||
request_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[int | None] = mapped_column(Integer)
|
||||
debug_session_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# A DOWNLOAD job's id (download_jobs) — not a background run, which is `run_id`.
|
||||
job_id: Mapped[int | None] = mapped_column(Integer)
|
||||
# The background run (job_runs) that logged it. No foreign key, as `user_id`: the writer runs
|
||||
# behind the event, and a run reaped meanwhile must not fail the batch its lines travel in.
|
||||
run_id: Mapped[int | None] = mapped_column(BigInteger)
|
||||
version: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
@@ -689,15 +623,44 @@ class AppState(Base):
|
||||
setup_token_hash: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class SchedulerSetting(Base):
|
||||
"""Admin override for a scheduler job's run interval (minutes). One row per job id;
|
||||
absent = use the env/config default. DB-backed so it's editable from the admin UI at
|
||||
runtime without a redeploy (env stays a fallback)."""
|
||||
class JobRun(Base):
|
||||
"""One run of a background job — scheduled, an admin's "run now", or a user's ad-hoc work on one
|
||||
target — written when it starts and again when it ends (app/jobruns.py), each in its own
|
||||
committed session, so the history survives the job's own rollback and a restart. A row left
|
||||
`running` by a process that died is set `interrupted` at the next startup.
|
||||
|
||||
__tablename__ = "scheduler_settings"
|
||||
`quota_units` is what the run spent of the YouTube budget: the `quota_events` rows carrying its
|
||||
id, summed when it ends. `summary` is the English one-line fallback; `result` / `failure` are the
|
||||
same outcome as data, which the client names in the reader's language."""
|
||||
|
||||
job_id: Mapped[str] = mapped_column(String(40), primary_key=True)
|
||||
interval_minutes: Mapped[int] = mapped_column(Integer)
|
||||
__tablename__ = "job_runs"
|
||||
__table_args__ = (
|
||||
Index("ix_job_runs_job_started", "job", "started_at"),
|
||||
CheckConstraint(f"status IN ({sql_in(RunStatus)})", name="ck_job_runs_status"),
|
||||
CheckConstraint(f"trigger IN ({sql_in(RunTrigger)})", name="ck_job_runs_trigger"),
|
||||
# Only a run still going has no end.
|
||||
CheckConstraint(
|
||||
f"(status = '{RunStatus.RUNNING}') = (finished_at IS NULL)", name="ck_job_runs_finished"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
job: Mapped[str] = mapped_column(String(40))
|
||||
trigger: Mapped[str] = mapped_column(String(16))
|
||||
# Who started it: the admin of a manual run, the user of an ad-hoc one; NULL = the schedule.
|
||||
actor_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
# What an ad-hoc run worked on (a channel id); NULL for the recurring jobs.
|
||||
target: Mapped[str | None] = mapped_column(String(128))
|
||||
status: Mapped[str] = mapped_column(String(16))
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
result: Mapped[dict | None] = mapped_column(JSONB)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
failure: Mapped[dict | None] = mapped_column(JSONB)
|
||||
quota_units: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class SystemConfig(Base):
|
||||
@@ -718,79 +681,6 @@ class SystemConfig(Base):
|
||||
)
|
||||
|
||||
|
||||
class Playlist(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A per-user named, ordered collection of videos from the shared catalog.
|
||||
|
||||
`kind`: 'user' (normal) or 'watch_later' (built-in, undeletable, one per user).
|
||||
`source`: 'local' (created here) or 'youtube' (mirrored from the user's YouTube
|
||||
playlists). `yt_playlist_id` links a mirrored/exported playlist to its YouTube id.
|
||||
`dirty` marks a YouTube-linked playlist with local edits not yet pushed, so the read
|
||||
sync won't clobber them. (source/yt_playlist_id/dirty are forward-looking for the YT
|
||||
sync phases; the foundation phase only uses local playlists.)"""
|
||||
|
||||
__tablename__ = "playlists"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
source: Mapped[str] = mapped_column(String(16), default="local", server_default="local")
|
||||
yt_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# True when the current items/name differ from `synced_fingerprint` (unpushed edits).
|
||||
dirty: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# SHA-256 of name + ordered video ids as last synced from / pushed to YouTube; the
|
||||
# baseline `dirty` is derived against. NULL = no baseline yet (treated as dirty).
|
||||
synced_fingerprint: Mapped[str | None] = mapped_column(String)
|
||||
# JSON list of the ordered video ids as of the last sync — the 3-way-merge baseline the
|
||||
# two-way YouTube sync classifies each item against (local-add vs YT-add vs real delete).
|
||||
# NULL = no baseline yet; the first merge then does a lossless union (never deletes).
|
||||
synced_video_ids: Mapped[str | None] = mapped_column(Text)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
items: Mapped[list["PlaylistItem"]] = relationship(
|
||||
back_populates="playlist",
|
||||
cascade="all, delete-orphan",
|
||||
order_by="PlaylistItem.position",
|
||||
)
|
||||
|
||||
|
||||
class PlaylistItem(Base):
|
||||
"""A video's membership in a playlist, with its order within that playlist."""
|
||||
|
||||
__tablename__ = "playlist_items"
|
||||
__table_args__ = (UniqueConstraint("playlist_id", "video_id", name="uq_playlist_video"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
playlist_id: Mapped[int] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
|
||||
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
playlist: Mapped["Playlist"] = relationship(back_populates="items")
|
||||
|
||||
|
||||
class SavedView(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A per-user named snapshot of a module's filter/sort state (a "smart view").
|
||||
|
||||
`source` names the module the view belongs to ("feed" | "plex"); `filters` is that module's
|
||||
serialized filter JSON the frontend applies with one click. `position` orders them in the sidebar
|
||||
(drag-reorder, per source); `is_default` marks the one view applied automatically on load — at most
|
||||
one per (user, source), enforced by the API on set. The name is unique per (user, source)."""
|
||||
|
||||
__tablename__ = "saved_views"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "source", "name", name="uq_user_source_saved_view"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
source: Mapped[str] = mapped_column(String(16), default="feed", server_default="feed")
|
||||
name: Mapped[str] = mapped_column(String(80))
|
||||
filters: Mapped[dict] = mapped_column(JSON)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
is_default: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
class Notification(Base):
|
||||
"""A durable, server-backed per-user notification (the inbox center, phase 1).
|
||||
|
||||
@@ -818,141 +708,6 @@ class Notification(Base):
|
||||
)
|
||||
|
||||
|
||||
class MediaAsset(Base, TimestampMixin):
|
||||
"""A physical downloaded media file — the SHARED cache layer of the download center.
|
||||
|
||||
Mirrors the app's "shared catalog, private per-user state" model: one row per unique
|
||||
``(source_kind, source_ref, format_sig)`` is the global cache. When a second user requests
|
||||
the same video with a format that hashes to the same `format_sig`, their `download_job`
|
||||
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
||||
shared; per-user edited derivatives (phase 2) are never cached here.
|
||||
|
||||
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
|
||||
system-decided and bound to the source id — never renamed (the user's custom name is display
|
||||
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
||||
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
|
||||
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
|
||||
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
|
||||
|
||||
__tablename__ = "media_assets"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"source_kind", "source_ref", "format_sig", name="uq_media_asset_cache_key"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512)) # video id, or full URL for ad-hoc
|
||||
format_sig: Mapped[str] = mapped_column(String(64)) # hash of the output-affecting spec
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="pending", server_default="pending", index=True
|
||||
) # "pending" | "ready" | "error"
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
|
||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
||||
# card / watch page / link preview can still show an image. Relative to files_root.
|
||||
poster_path: Mapped[str | None] = mapped_column(Text)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||
width: Mapped[int | None] = mapped_column(Integer)
|
||||
height: Mapped[int | None] = mapped_column(Integer)
|
||||
container: Mapped[str | None] = mapped_column(String(16))
|
||||
vcodec: Mapped[str | None] = mapped_column(String(32))
|
||||
acodec: Mapped[str | None] = mapped_column(String(32))
|
||||
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
|
||||
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
|
||||
# still pending (kind is known once the file is probed at finalize). It is the single stored home
|
||||
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
|
||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||
title: Mapped[str | None] = mapped_column(Text)
|
||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
# yt-dlp's channel/uploader page URL (channel_url|uploader_url) — lets an auto-filled channel
|
||||
# render as a clickable link. Null for sources without one (e.g. reddit) or older rows.
|
||||
uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
upload_date: Mapped[date | None] = mapped_column(Date)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
||||
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
||||
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
|
||||
# fall back to a URL derived from source_kind+source_ref).
|
||||
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
|
||||
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
|
||||
sha256: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
@validates("rel_path")
|
||||
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
|
||||
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
|
||||
self.sha256 = None
|
||||
return rel_path
|
||||
|
||||
|
||||
class DownloadProfile(Base, TimestampMixin):
|
||||
"""A named download preset — a reusable format profile.
|
||||
|
||||
`user_id` NULL + `is_builtin` marks the shipped presets (Best MP4, 1080p, audio-only, …);
|
||||
a user's saved custom profiles carry their `user_id`. `spec` is the format definition
|
||||
(mode av/v/a, max_height, container, audio_format, codec prefs, embed subs/chapters/
|
||||
thumbnail, sponsorblock) that `downloads.formats` turns into a yt-dlp selector + a cache
|
||||
`format_sig`."""
|
||||
|
||||
__tablename__ = "download_profiles"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
) # NULL for a builtin preset
|
||||
name: Mapped[str] = mapped_column(String(80))
|
||||
spec: Mapped[dict] = mapped_column(JSON)
|
||||
is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class FileFolder(Base, TimestampMixin):
|
||||
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
|
||||
|
||||
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
|
||||
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
|
||||
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
|
||||
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
|
||||
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
|
||||
|
||||
__tablename__ = "file_folders"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||
Index(
|
||||
"uq_file_folder_own_name",
|
||||
"user_id",
|
||||
func.lower(text("name")),
|
||||
unique=True,
|
||||
postgresql_where=text("name IS NOT NULL"),
|
||||
),
|
||||
# One Hub folder per user.
|
||||
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
|
||||
CheckConstraint(
|
||||
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
||||
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
|
||||
name="ck_file_folder_kind",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
|
||||
# "site:" + a host name (DNS caps one at 253 characters).
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
retention_days: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
|
||||
class ApiToken(Base, TimestampMixin):
|
||||
"""A user's token for a machine to reach their Files library with (`app.apitokens`): sent as
|
||||
`Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its
|
||||
@@ -994,208 +749,6 @@ class ApiTokenGrant(Base):
|
||||
can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
def _ready_on_insert(context) -> datetime | None:
|
||||
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
|
||||
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
|
||||
done = context.get_current_parameters().get("status") == "done"
|
||||
return datetime.now(timezone.utc) if done else None
|
||||
|
||||
|
||||
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||
|
||||
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
|
||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
|
||||
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
|
||||
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
|
||||
`display_name` is the user's own name for the file (display + the Content-Disposition on
|
||||
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
|
||||
(SET NULL if the asset is GC'd — the job then shows as expired).
|
||||
|
||||
Phase 2 (editor): `job_kind='edit'` jobs derive a new per-user clip from an already-downloaded
|
||||
ready job. They still hang off a `MediaAsset` (with `source_kind='edit'`, a per-user cache key)
|
||||
so all the file-serve/ref-count/GC/quota machinery is reused; the worker branches on the
|
||||
asset's `source_kind` and runs ffmpeg instead of yt-dlp. `source_job_id`/`source_asset_id`
|
||||
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
|
||||
|
||||
__tablename__ = "download_jobs"
|
||||
__table_args__ = (
|
||||
Index("ix_download_jobs_claim", "status", "queue_pos"),
|
||||
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
|
||||
CheckConstraint(
|
||||
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512))
|
||||
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
|
||||
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
|
||||
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
|
||||
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
||||
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
||||
source_job_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="SET NULL")
|
||||
)
|
||||
source_asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL")
|
||||
)
|
||||
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
|
||||
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
|
||||
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
|
||||
edit_spec: Mapped[dict | None] = mapped_column(JSON)
|
||||
profile_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||
)
|
||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
||||
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
||||
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
extra_links: Mapped[list | None] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="queued", server_default="queued", index=True
|
||||
) # queued | running | paused | done | error | canceled
|
||||
progress: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
speed_bps: Mapped[int | None] = mapped_column(BigInteger)
|
||||
eta_s: Mapped[int | None] = mapped_column(Integer)
|
||||
phase: Mapped[str | None] = mapped_column(String(32))
|
||||
error: Mapped[str | None] = mapped_column(Text) # raw yt-dlp/ffmpeg text (technical detail)
|
||||
error_code: Mapped[str | None] = mapped_column(
|
||||
String(24)
|
||||
) # category, see downloads.errors.classify
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
|
||||
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
|
||||
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
|
||||
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
||||
)
|
||||
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
|
||||
# traces its origin through its parent, and that parent may be deleted or expire later — the
|
||||
# file must still know which source folder it may go back to. NULL only for a file placed before
|
||||
# the column existed outside a source folder; `folders.source_key_for` works those out.
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# When the file became done — what the library's "Added" means: a download that failed for days
|
||||
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
|
||||
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
|
||||
ready_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), default=_ready_on_insert
|
||||
)
|
||||
|
||||
|
||||
class DownloadShare(Base, TimestampMixin):
|
||||
"""A user→user grant: `to_user` may access `from_user`'s downloaded file (private by
|
||||
default). Since assets are already deduped/shared on disk, a share is just an ACL row."""
|
||||
|
||||
__tablename__ = "download_shares"
|
||||
__table_args__ = (UniqueConstraint("job_id", "to_user_id", name="uq_download_share"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int] = mapped_column(ForeignKey("download_jobs.id", ondelete="CASCADE"))
|
||||
from_user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
|
||||
class DownloadQuota(Base, UpdatedAtMixin):
|
||||
"""Per-user download limits (admin-managed). A row exists only when an admin customizes a
|
||||
user; otherwise the sysconfig `download_default_*` values apply. `unlimited` bypasses the
|
||||
byte cap entirely (e.g. for the admin's own account)."""
|
||||
|
||||
__tablename__ = "download_quotas"
|
||||
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
max_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
max_concurrent: Mapped[int] = mapped_column(Integer)
|
||||
max_jobs: Mapped[int] = mapped_column(Integer)
|
||||
unlimited: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
class DownloadLink(Base, TimestampMixin):
|
||||
"""A public capability link to one download, or to one of its owner's own folders — the "share
|
||||
by link" surface.
|
||||
|
||||
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
|
||||
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
|
||||
the folder at the time it is opened. The token itself is never stored as written: a link is
|
||||
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
|
||||
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
|
||||
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
|
||||
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
|
||||
grants a *registered* user access via the in-app "Shared with me" list."""
|
||||
|
||||
__tablename__ = "download_links"
|
||||
__table_args__ = (
|
||||
# A link opens exactly one thing: a file or a folder.
|
||||
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
password_hash: Mapped[str | None] = mapped_column(Text)
|
||||
# Bumped whenever the password changes; folded into the signed watch grant so rotating the
|
||||
# password immediately invalidates any grants issued under the old one (instead of them living
|
||||
# out the 6h grant TTL).
|
||||
password_version: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
|
||||
|
||||
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
|
||||
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
|
||||
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
|
||||
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
|
||||
the routes are write-only (host + updated_at only). See `downloads.credentials`.
|
||||
|
||||
Two load-bearing security controls live around this row, NOT on it:
|
||||
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
|
||||
user id, so the authed asset is private to that user and can never be cache-served to
|
||||
another user (`downloads.credentials.cookie_salt`).
|
||||
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
|
||||
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
|
||||
can't exfiltrate the user's session to a look-alike host.
|
||||
|
||||
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
|
||||
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
|
||||
|
||||
__tablename__ = "download_credentials"
|
||||
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
host_key: Mapped[str] = mapped_column(String(64))
|
||||
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
|
||||
|
||||
# --- Plex integration (optional) -------------------------------------------------------------
|
||||
# A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is
|
||||
# from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states
|
||||
@@ -1630,52 +1183,3 @@ class PlexPlaylistItem(Base):
|
||||
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||
|
||||
|
||||
class ComfortPin(Base, TimestampMixin):
|
||||
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X".
|
||||
|
||||
Deliberately NOT a boolean on `VideoState`/`PlexState`, and NOT a polymorphic `(kind, ref)` string
|
||||
pair. A flag cannot ORDER a shelf and cannot hold a channel or a show — and comfort media is
|
||||
usually "put on X", a series or a channel, not one particular video. A string ref would order and
|
||||
hold anything, but it would also lose the foreign key: a Plex item dropped from the library, or an
|
||||
unsubscribed channel, would leave a dead tile on the one surface whose whole job is to be
|
||||
reliable. So instead: one nullable CASCADE-ing FK per pinnable kind, and a check constraint that
|
||||
exactly one of them is set.
|
||||
|
||||
That constraint is the house rule applied to a five-way union — the invariant lives in the schema,
|
||||
not in a comment every insert site has to remember. `kind`/`ref` below derive FROM the columns, so
|
||||
there is no second vocabulary to keep in sync.
|
||||
|
||||
NULLs are DISTINCT in a Postgres unique index, so the five per-kind unique constraints each stop a
|
||||
double-pin of the same thing without colliding across kinds."""
|
||||
|
||||
__tablename__ = "comfort_pins"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"num_nonnulls(video_id, channel_id, plex_item_id, plex_show_id, playlist_id) = 1",
|
||||
name="ck_comfort_pin_one_ref",
|
||||
),
|
||||
UniqueConstraint("user_id", "video_id", name="uq_comfort_pin_video"),
|
||||
UniqueConstraint("user_id", "channel_id", name="uq_comfort_pin_channel"),
|
||||
UniqueConstraint("user_id", "plex_item_id", name="uq_comfort_pin_plex_item"),
|
||||
UniqueConstraint("user_id", "plex_show_id", name="uq_comfort_pin_plex_show"),
|
||||
UniqueConstraint("user_id", "playlist_id", name="uq_comfort_pin_playlist"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
video_id: Mapped[str | None] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str | None] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||
plex_item_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_items.id", ondelete="CASCADE")
|
||||
)
|
||||
plex_show_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_shows.id", ondelete="CASCADE")
|
||||
)
|
||||
playlist_id: Mapped[int | None] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
|
||||
|
||||
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
|
||||
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||
|
||||
+11
-3
@@ -16,7 +16,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.db import SessionLocal
|
||||
from app.models import ApiQuotaUsage, QuotaEvent
|
||||
|
||||
@@ -204,6 +204,14 @@ def record_usage(units: int) -> None:
|
||||
)
|
||||
with SessionLocal() as qs:
|
||||
qs.execute(stmt)
|
||||
# Audit detail: who/what spent it (per-user attribution; NULL actor = system).
|
||||
qs.add(QuotaEvent(user_id=_actor_id.get(), action=_action.get(), units=units))
|
||||
# Audit detail: who/what spent it (per-user attribution; NULL actor = system), and the
|
||||
# background run it belongs to, so the run's row can say what it cost.
|
||||
qs.add(
|
||||
QuotaEvent(
|
||||
user_id=_actor_id.get(),
|
||||
action=_action.get(),
|
||||
units=units,
|
||||
job_run_id=logs.current_run_id(),
|
||||
)
|
||||
)
|
||||
qs.commit()
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Reader's tables: what a user does with the shared catalog — where they are in a video
|
||||
(`VideoState`), their playlists, their saved views, their labels on channels (`Tag`, `ChannelTag`)
|
||||
and the comfort shelf on Home. Only this unit writes them; another unit reads them along a declared
|
||||
edge, or asks this unit's service."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.db import Base
|
||||
from app.models import TimestampMixin, UpdatedAtMixin
|
||||
|
||||
|
||||
class Tag(Base, TimestampMixin):
|
||||
"""A label. System tags (user_id NULL) are computed automatically and shared by all
|
||||
users; user tags (user_id set) are personal. category groups them in the UI."""
|
||||
|
||||
__tablename__ = "tags"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(64))
|
||||
color: Mapped[str | None] = mapped_column(String(16))
|
||||
# language | topic | other
|
||||
category: Mapped[str] = mapped_column(String(16), default="other", server_default="other")
|
||||
|
||||
|
||||
class ChannelTag(Base):
|
||||
"""Links a tag to a channel. source='auto' rows are managed by the auto-tagger and
|
||||
regenerated freely; source='user' rows are never touched by the system."""
|
||||
|
||||
__tablename__ = "channel_tags"
|
||||
__table_args__ = (UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
tag_id: Mapped[int] = mapped_column(ForeignKey("tags.id", ondelete="CASCADE"), index=True)
|
||||
# NULL = system/auto link visible to everyone; set = a specific user's link.
|
||||
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
|
||||
confidence: Mapped[float | None] = mapped_column(Float)
|
||||
|
||||
|
||||
class VideoState(Base, UpdatedAtMixin):
|
||||
"""Per-user watch state for a video. Rows exist only for non-default states; the
|
||||
absence of a row means 'new/unseen'."""
|
||||
|
||||
__tablename__ = "video_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_video"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||
# new | watched | hidden ("saved" is now Watch-later playlist membership, not a status)
|
||||
status: Mapped[str] = mapped_column(String(12), default="new", server_default="new")
|
||||
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Resume position (seconds into the video) for the in-app player. Orthogonal to
|
||||
# status: a partially-watched video keeps status "new" but a non-zero position, so
|
||||
# it can render a progress bar and match the "in progress" feed filter. 0 = not
|
||||
# started / finished (trivially-early and near-finished positions are not stored).
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(
|
||||
Integer, default=0, server_default="0", nullable=False
|
||||
)
|
||||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class Playlist(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A per-user named, ordered collection of videos from the shared catalog.
|
||||
|
||||
`kind`: 'user' (normal) or 'watch_later' (built-in, undeletable, one per user).
|
||||
`source`: 'local' (created here) or 'youtube' (mirrored from the user's YouTube
|
||||
playlists). `yt_playlist_id` links a mirrored/exported playlist to its YouTube id.
|
||||
`dirty` marks a YouTube-linked playlist with local edits not yet pushed, so the read
|
||||
sync won't clobber them. (source/yt_playlist_id/dirty are forward-looking for the YT
|
||||
sync phases; the foundation phase only uses local playlists.)"""
|
||||
|
||||
__tablename__ = "playlists"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
source: Mapped[str] = mapped_column(String(16), default="local", server_default="local")
|
||||
yt_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# True when the current items/name differ from `synced_fingerprint` (unpushed edits).
|
||||
dirty: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# SHA-256 of name + ordered video ids as last synced from / pushed to YouTube; the
|
||||
# baseline `dirty` is derived against. NULL = no baseline yet (treated as dirty).
|
||||
synced_fingerprint: Mapped[str | None] = mapped_column(String)
|
||||
# JSON list of the ordered video ids as of the last sync — the 3-way-merge baseline the
|
||||
# two-way YouTube sync classifies each item against (local-add vs YT-add vs real delete).
|
||||
# NULL = no baseline yet; the first merge then does a lossless union (never deletes).
|
||||
synced_video_ids: Mapped[str | None] = mapped_column(Text)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
items: Mapped[list["PlaylistItem"]] = relationship(
|
||||
back_populates="playlist",
|
||||
cascade="all, delete-orphan",
|
||||
order_by="PlaylistItem.position",
|
||||
)
|
||||
|
||||
|
||||
class PlaylistItem(Base):
|
||||
"""A video's membership in a playlist, with its order within that playlist."""
|
||||
|
||||
__tablename__ = "playlist_items"
|
||||
__table_args__ = (UniqueConstraint("playlist_id", "video_id", name="uq_playlist_video"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
playlist_id: Mapped[int] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
|
||||
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
playlist: Mapped["Playlist"] = relationship(back_populates="items")
|
||||
|
||||
|
||||
class SavedView(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A per-user named snapshot of a module's filter/sort state (a "smart view").
|
||||
|
||||
`source` names the module the view belongs to ("feed" | "plex"); `filters` is that module's
|
||||
serialized filter JSON the frontend applies with one click. `position` orders them in the sidebar
|
||||
(drag-reorder, per source); `is_default` marks the one view applied automatically on load — at most
|
||||
one per (user, source), enforced by the API on set. The name is unique per (user, source)."""
|
||||
|
||||
__tablename__ = "saved_views"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "source", "name", name="uq_user_source_saved_view"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
source: Mapped[str] = mapped_column(String(16), default="feed", server_default="feed")
|
||||
name: Mapped[str] = mapped_column(String(80))
|
||||
filters: Mapped[dict] = mapped_column(JSON)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
is_default: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
class ComfortPin(Base, TimestampMixin):
|
||||
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X".
|
||||
|
||||
Deliberately NOT a boolean on `VideoState`/`PlexState`, and NOT a polymorphic `(kind, ref)` string
|
||||
pair. A flag cannot ORDER a shelf and cannot hold a channel or a show — and comfort media is
|
||||
usually "put on X", a series or a channel, not one particular video. A string ref would order and
|
||||
hold anything, but it would also lose the foreign key: a Plex item dropped from the library, or an
|
||||
unsubscribed channel, would leave a dead tile on the one surface whose whole job is to be
|
||||
reliable. So instead: one nullable CASCADE-ing FK per pinnable kind, and a check constraint that
|
||||
exactly one of them is set.
|
||||
|
||||
That constraint is the house rule applied to a five-way union — the invariant lives in the schema,
|
||||
not in a comment every insert site has to remember. `kind`/`ref` below derive FROM the columns, so
|
||||
there is no second vocabulary to keep in sync.
|
||||
|
||||
NULLs are DISTINCT in a Postgres unique index, so the five per-kind unique constraints each stop a
|
||||
double-pin of the same thing without colliding across kinds."""
|
||||
|
||||
__tablename__ = "comfort_pins"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"num_nonnulls(video_id, channel_id, plex_item_id, plex_show_id, playlist_id) = 1",
|
||||
name="ck_comfort_pin_one_ref",
|
||||
),
|
||||
UniqueConstraint("user_id", "video_id", name="uq_comfort_pin_video"),
|
||||
UniqueConstraint("user_id", "channel_id", name="uq_comfort_pin_channel"),
|
||||
UniqueConstraint("user_id", "plex_item_id", name="uq_comfort_pin_plex_item"),
|
||||
UniqueConstraint("user_id", "plex_show_id", name="uq_comfort_pin_plex_show"),
|
||||
UniqueConstraint("user_id", "playlist_id", name="uq_comfort_pin_playlist"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
video_id: Mapped[str | None] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str | None] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||
plex_item_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_items.id", ondelete="CASCADE")
|
||||
)
|
||||
plex_show_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_shows.id", ondelete="CASCADE")
|
||||
)
|
||||
playlist_id: Mapped[int | None] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
|
||||
|
||||
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
|
||||
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and
|
||||
for the other units that ask about them.
|
||||
|
||||
Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as
|
||||
a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and
|
||||
length, whether a video exists); another unit asks the questions at the end of this module rather
|
||||
than joining Reader's tables itself.
|
||||
"""
|
||||
|
||||
from sqlalchemy import ColumnElement, and_, func, or_, select
|
||||
from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
|
||||
|
||||
from app.feedquery import feed_columns
|
||||
from app.models import Channel, User, Video
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
# --- catalog, read along the edge ------------------------------------------------------------
|
||||
|
||||
|
||||
def video(db: Session, video_id: str) -> Video | None:
|
||||
return db.get(Video, video_id)
|
||||
|
||||
|
||||
# --- tags ------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def tags_with_counts(db: Session, user: User) -> list[tuple[Tag, int]]:
|
||||
"""The tags this user sees (the shared system ones and their own), each with how many channel
|
||||
links it has, by category then name."""
|
||||
return [
|
||||
(tag, count)
|
||||
for tag, count in db.execute(
|
||||
select(Tag, func.count(ChannelTag.id))
|
||||
.outerjoin(ChannelTag, ChannelTag.tag_id == Tag.id)
|
||||
.where(or_(Tag.user_id.is_(None), owned_by(Tag.user_id, user)))
|
||||
.group_by(Tag.id)
|
||||
.order_by(Tag.category, Tag.name)
|
||||
).tuples()
|
||||
]
|
||||
|
||||
|
||||
def own_tag(db: Session, user: User, tag_id: int) -> Tag | None:
|
||||
"""One of the user's own tags. A system tag (no owner) is nobody's to edit, so it reads as
|
||||
missing too."""
|
||||
return owned_row(db, Tag, tag_id, user)
|
||||
|
||||
|
||||
# --- saved views -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def views(db: Session, user: User, source: str) -> list[SavedView]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(SavedView)
|
||||
.where(owned_by(SavedView.user_id, user), SavedView.source == source)
|
||||
.order_by(SavedView.position, SavedView.id)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def all_views(db: Session, user: User) -> list[SavedView]:
|
||||
return list(db.execute(select(SavedView).where(owned_by(SavedView.user_id, user))).scalars())
|
||||
|
||||
|
||||
def own_view(db: Session, user: User, view_id: int) -> SavedView | None:
|
||||
return owned_row(db, SavedView, view_id, user)
|
||||
|
||||
|
||||
# --- watch state -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def video_state(db: Session, user: User, video_id: str) -> VideoState | None:
|
||||
return db.execute(
|
||||
select(VideoState).where(
|
||||
owned_by(VideoState.user_id, user), VideoState.video_id == video_id
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
# --- playlists -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def own_playlist(
|
||||
db: Session, user: User, playlist_id: int, *, lock: bool = False
|
||||
) -> Playlist | None:
|
||||
return owned_row(db, Playlist, playlist_id, user, lock=lock)
|
||||
|
||||
|
||||
def playlists_for(db: Session, user: User) -> list[Playlist]:
|
||||
"""The user's playlists in their rail order."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(Playlist)
|
||||
.where(owned_by(Playlist.user_id, user))
|
||||
.order_by(Playlist.position, Playlist.created_at)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def watch_later(db: Session, user: User) -> Playlist | None:
|
||||
return db.scalar(
|
||||
select(Playlist).where(owned_by(Playlist.user_id, user), Playlist.kind == "watch_later")
|
||||
)
|
||||
|
||||
|
||||
def item_counts(db: Session, ids: list[int]) -> dict[int, int]:
|
||||
return dict(
|
||||
db.execute(
|
||||
select(PlaylistItem.playlist_id, func.count())
|
||||
.where(PlaylistItem.playlist_id.in_(ids))
|
||||
.group_by(PlaylistItem.playlist_id)
|
||||
)
|
||||
.tuples()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def durations(db: Session, ids: list[int]) -> dict[int, int]:
|
||||
"""Each playlist's running time: the sum of its videos' lengths."""
|
||||
return {
|
||||
pid: int(total or 0)
|
||||
for pid, total in db.execute(
|
||||
select(PlaylistItem.playlist_id, func.coalesce(func.sum(Video.duration_seconds), 0))
|
||||
.join(Video, Video.id == PlaylistItem.video_id)
|
||||
.where(PlaylistItem.playlist_id.in_(ids))
|
||||
.group_by(PlaylistItem.playlist_id)
|
||||
).tuples()
|
||||
}
|
||||
|
||||
|
||||
def containing(db: Session, ids: list[int], video_id: str) -> set[int]:
|
||||
"""Which of these playlists already hold the video (the add-to-playlist popover's ticks)."""
|
||||
return set(
|
||||
db.execute(
|
||||
select(PlaylistItem.playlist_id).where(
|
||||
PlaylistItem.playlist_id.in_(ids), PlaylistItem.video_id == video_id
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def covers(db: Session, ids: list[int]) -> dict[int, str | None]:
|
||||
"""Each playlist's cover: its first item's thumbnail, for every list in ONE window query rather
|
||||
than a lookup per playlist."""
|
||||
rn = (
|
||||
func.row_number()
|
||||
.over(partition_by=PlaylistItem.playlist_id, order_by=PlaylistItem.position)
|
||||
.label("rn")
|
||||
)
|
||||
first_items = (
|
||||
select(PlaylistItem.playlist_id.label("pid"), Video.thumbnail_url.label("thumb"), rn)
|
||||
.join(Video, Video.id == PlaylistItem.video_id)
|
||||
.where(PlaylistItem.playlist_id.in_(ids))
|
||||
.subquery()
|
||||
)
|
||||
return dict(
|
||||
db.execute(select(first_items.c.pid, first_items.c.thumb).where(first_items.c.rn == 1))
|
||||
.tuples()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None]:
|
||||
"""One playlist's item count, running time and cover. An outer join, so an empty list still
|
||||
answers (0, 0, None)."""
|
||||
count, total = db.execute(
|
||||
select(func.count(PlaylistItem.id), func.coalesce(func.sum(Video.duration_seconds), 0))
|
||||
.select_from(PlaylistItem)
|
||||
.outerjoin(Video, Video.id == PlaylistItem.video_id)
|
||||
.where(PlaylistItem.playlist_id == playlist_id)
|
||||
).one()
|
||||
cover = db.scalar(
|
||||
select(Video.thumbnail_url)
|
||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||
.where(PlaylistItem.playlist_id == playlist_id)
|
||||
.order_by(PlaylistItem.position)
|
||||
.limit(1)
|
||||
)
|
||||
return count or 0, total or 0, cover
|
||||
|
||||
|
||||
def playlist_rows(db: Session, user: User, playlist_id: int) -> list:
|
||||
"""A playlist's videos as feed rows (the same columns the feed serializes), in list order."""
|
||||
state = aliased(VideoState)
|
||||
return list(
|
||||
db.execute(
|
||||
select(*feed_columns(user, state))
|
||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(state, and_(state.video_id == Video.id, owned_by(state.user_id, user)))
|
||||
.where(PlaylistItem.playlist_id == playlist_id)
|
||||
# `id` breaks a position tie the same way on every read (the renumbering add does too).
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def items(db: Session, playlist_id: int) -> list[PlaylistItem]:
|
||||
"""A playlist's item rows, in list order."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(PlaylistItem)
|
||||
.where(PlaylistItem.playlist_id == playlist_id)
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def item(db: Session, playlist_id: int, video_id: str) -> PlaylistItem | None:
|
||||
return db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == playlist_id, PlaylistItem.video_id == video_id
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --- for the other units ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def kept_by_a_reader(video_id: InstrumentedAttribute[str]) -> ColumnElement[bool]:
|
||||
"""A video someone holds: watched, in progress or hidden (a `VideoState`), or in one of their
|
||||
playlists. The catalog's cleanup jobs keep such a video; as a clause, so their DELETE stays
|
||||
one statement."""
|
||||
return or_(
|
||||
select(VideoState.id).where(VideoState.video_id == video_id).exists(),
|
||||
select(PlaylistItem.id).where(PlaylistItem.video_id == video_id).exists(),
|
||||
)
|
||||
|
||||
|
||||
def playlisted_by(db: Session, video_ids: list[str]) -> list[tuple[int, str]]:
|
||||
"""(user id, video id) for every user who has one of these videos in a playlist."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(Playlist.user_id, PlaylistItem.video_id)
|
||||
.join(PlaylistItem, PlaylistItem.playlist_id == Playlist.id)
|
||||
.where(PlaylistItem.video_id.in_(video_ids))
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def has_auto_tags(channel_id: InstrumentedAttribute[str]) -> ColumnElement[bool]:
|
||||
"""A channel the auto-tagger has already labelled, as a clause for its "only the missing" scan."""
|
||||
return (
|
||||
select(ChannelTag.id)
|
||||
.where(
|
||||
ChannelTag.channel_id == channel_id,
|
||||
ChannelTag.source == "auto",
|
||||
ChannelTag.user_id.is_(None),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
def view_tag_ids(filters: object) -> list[int]:
|
||||
"""The tag ids a stored view's filters name; anything else in that slot names none."""
|
||||
tags = filters.get("tags") if isinstance(filters, dict) else None
|
||||
if not isinstance(tags, list):
|
||||
return []
|
||||
return [t for t in tags if type(t) is int] # `type`, not isinstance: True is not a tag id
|
||||
|
||||
|
||||
def tags_in_feed_views(db: Session) -> set[int]:
|
||||
"""Every tag id any user's saved feed view filters on. Read defensively (`view_tag_ids`): a view
|
||||
stored before its route checked the shape must not raise for everyone's job."""
|
||||
return {
|
||||
t
|
||||
for (filters,) in db.execute(select(SavedView.filters).where(SavedView.source == "feed"))
|
||||
for t in view_tag_ids(filters)
|
||||
}
|
||||
@@ -0,0 +1,466 @@
|
||||
"""Reader's writes: a user's labels and saved views, where they are in a video, their playlists'
|
||||
items and order — and the system tags the catalog's auto-tagger hands over.
|
||||
|
||||
Every function here flushes and leaves the commit to its caller (the handler, or the job that owns
|
||||
its session). A write that must not double up keeps its guard where it is written: an upsert on the
|
||||
unique key, a row lock taken before a presence check. The comfort shelf's writes are Home's
|
||||
(`discover.pins`); a playlist's YouTube sync is `sync.playlists`.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import exists, func, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import User
|
||||
from app.reader import queries
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
||||
from app.sync.playlists import lock_playlists, recompute_dirty
|
||||
from app.userscope import owned_by
|
||||
|
||||
TAG_CATEGORIES = ("language", "topic", "other")
|
||||
VIEW_NAME_MAX = 80
|
||||
|
||||
# --- tags ------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def create_tag(db: Session, user: User, name: str, category: str, color: str | None) -> Tag:
|
||||
"""A tag of the user's own. An unknown category falls back to "other" rather than refusing. A
|
||||
name the user already has raises `IntegrityError` at the flush."""
|
||||
tag = Tag(
|
||||
user_id=user.id,
|
||||
name=name,
|
||||
color=color,
|
||||
category=category if category in TAG_CATEGORIES else "other",
|
||||
)
|
||||
db.add(tag)
|
||||
db.flush()
|
||||
return tag
|
||||
|
||||
|
||||
def update_tag(tag: Tag, *, sent: set[str], name: str | None, color: str | None) -> None:
|
||||
"""Change the fields the caller sent: `color` sent as None clears it."""
|
||||
if "name" in sent and name is not None:
|
||||
tag.name = name
|
||||
if "color" in sent:
|
||||
tag.color = color
|
||||
|
||||
|
||||
def delete_tag(db: Session, tag: Tag) -> None:
|
||||
db.delete(tag) # its channel links cascade
|
||||
db.flush()
|
||||
|
||||
|
||||
def ensure_system_tag(db: Session, name: str, category: str, color: str | None) -> Tag:
|
||||
"""The shared system tag of this name, made the first time a channel earns it."""
|
||||
tag = db.execute(
|
||||
select(Tag).where(Tag.user_id.is_(None), Tag.name == name)
|
||||
).scalar_one_or_none()
|
||||
if tag is None:
|
||||
tag = Tag(user_id=None, name=name, category=category, color=color)
|
||||
db.add(tag)
|
||||
db.flush()
|
||||
return tag
|
||||
|
||||
|
||||
def set_auto_tags(db: Session, channel_id: str, desired: dict[int, float | None]) -> None:
|
||||
"""Make a channel's automatic tag links exactly `desired` (tag id → confidence). Only the
|
||||
auto-tagger's own links move; a user's links on the channel are never touched."""
|
||||
existing = {
|
||||
ct.tag_id: ct
|
||||
for ct in db.execute(
|
||||
select(ChannelTag).where(
|
||||
ChannelTag.channel_id == channel_id,
|
||||
ChannelTag.user_id.is_(None),
|
||||
ChannelTag.source == "auto",
|
||||
)
|
||||
).scalars()
|
||||
}
|
||||
for tag_id, ct in existing.items():
|
||||
if tag_id not in desired:
|
||||
db.delete(ct)
|
||||
for tag_id, conf in desired.items():
|
||||
ct = existing.get(tag_id)
|
||||
if ct is None:
|
||||
db.add(
|
||||
ChannelTag(
|
||||
channel_id=channel_id,
|
||||
tag_id=tag_id,
|
||||
user_id=None,
|
||||
source="auto",
|
||||
confidence=conf,
|
||||
)
|
||||
)
|
||||
else:
|
||||
ct.confidence = conf
|
||||
db.flush()
|
||||
|
||||
|
||||
def prune_orphan_system_tags(db: Session) -> int:
|
||||
"""Delete the system tags no channel links to any more (after a mapping change) — but not one a
|
||||
saved feed view filters on. Views hold tag IDS, and a deleted tag comes back under a new id when
|
||||
a channel earns it again, so the view's filter would point at nothing; the feed drops an unknown
|
||||
id, and the view showed the whole feed under its old name. The kept ids are filtered here rather
|
||||
than in SQL: a `NOT IN` over every view's ids could pass the driver's bind-parameter limit."""
|
||||
in_views = queries.tags_in_feed_views(db)
|
||||
orphans = [
|
||||
tag
|
||||
for tag in db.execute(
|
||||
select(Tag).where(Tag.user_id.is_(None), ~exists().where(ChannelTag.tag_id == Tag.id))
|
||||
).scalars()
|
||||
if tag.id not in in_views
|
||||
]
|
||||
for tag in orphans:
|
||||
db.delete(tag)
|
||||
db.flush()
|
||||
return len(orphans)
|
||||
|
||||
|
||||
# --- saved views -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def create_view(db: Session, user: User, source: str, name: str, filters: dict) -> SavedView:
|
||||
"""A view appended after the user's others of the same module (each module numbers its own
|
||||
0..N). A name the user already has there raises `IntegrityError` at the flush."""
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(SavedView.position), -1)).where(
|
||||
owned_by(SavedView.user_id, user), SavedView.source == source
|
||||
)
|
||||
).scalar_one()
|
||||
view = SavedView(
|
||||
user_id=user.id,
|
||||
source=source,
|
||||
name=name[:VIEW_NAME_MAX],
|
||||
filters=filters,
|
||||
position=maxpos + 1,
|
||||
)
|
||||
db.add(view)
|
||||
db.flush()
|
||||
return view
|
||||
|
||||
|
||||
def update_view(
|
||||
db: Session,
|
||||
view: SavedView,
|
||||
*,
|
||||
sent: set[str],
|
||||
name: str | None = None,
|
||||
filters: dict | None = None,
|
||||
is_default: bool | None = None,
|
||||
) -> None:
|
||||
"""Change the fields the caller sent. Making a view the default clears the flag on the user's
|
||||
other views of the same module first: at most one default per (user, source)."""
|
||||
if "name" in sent and name is not None:
|
||||
view.name = name[:VIEW_NAME_MAX]
|
||||
if "filters" in sent and filters is not None:
|
||||
view.filters = filters
|
||||
if "is_default" in sent:
|
||||
make_default = bool(is_default)
|
||||
if make_default:
|
||||
db.execute(
|
||||
update(SavedView)
|
||||
.where(
|
||||
owned_by(SavedView.user_id, view.user_id),
|
||||
SavedView.source == view.source,
|
||||
SavedView.id != view.id,
|
||||
)
|
||||
.values(is_default=False)
|
||||
)
|
||||
view.is_default = make_default
|
||||
db.flush()
|
||||
|
||||
|
||||
def delete_view(db: Session, view: SavedView) -> None:
|
||||
db.delete(view)
|
||||
db.flush()
|
||||
|
||||
|
||||
def reorder_views(db: Session, user: User, ids: list[int]) -> None:
|
||||
"""Number the listed views 0..n in the listed order; ids that are not the user's are skipped."""
|
||||
by_id = {v.id: v for v in queries.all_views(db, user)}
|
||||
pos = 0
|
||||
for vid in ids:
|
||||
v = by_id.get(vid)
|
||||
if v is not None:
|
||||
v.position = pos
|
||||
pos += 1
|
||||
db.flush()
|
||||
|
||||
|
||||
# --- watch state -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def unmark(db: Session, user: User, video_id: str) -> None:
|
||||
"""Back to "new". A row that still holds a resume point keeps it (un-marking "watched" restores
|
||||
the in-progress state, not where the user left off); otherwise the row goes. The player fires
|
||||
this beside a checkpoint that may delete the same row: the caller's commit then raises
|
||||
`StaleDataError`, which is that lost race and nothing worse."""
|
||||
row = queries.video_state(db, user, video_id)
|
||||
if row is None:
|
||||
return
|
||||
if row.position_seconds:
|
||||
row.status = "new"
|
||||
row.watched_at = None
|
||||
else:
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
|
||||
|
||||
def mark(db: Session, user: User, video_id: str, status: str) -> None:
|
||||
""" "watched" or "hidden", as one atomic upsert on (user, video). The player marks "watched" at
|
||||
the end of playback at the instant a checkpoint may delete the in-progress row; a SELECT then
|
||||
UPDATE raised `StaleDataError` on that race, and an INSERT … ON CONFLICT is immune to it.
|
||||
"hidden" keeps any earlier `watched_at`."""
|
||||
now = datetime.now(timezone.utc)
|
||||
set_: dict = {"status": status, "updated_at": now}
|
||||
if status == "watched":
|
||||
set_["watched_at"] = now
|
||||
db.execute(
|
||||
pg_insert(VideoState)
|
||||
.values(
|
||||
user_id=user.id,
|
||||
video_id=video_id,
|
||||
status=status,
|
||||
watched_at=now if status == "watched" else None,
|
||||
)
|
||||
.on_conflict_do_update(constraint="uq_user_video", set_=set_)
|
||||
)
|
||||
|
||||
|
||||
def clear_state(db: Session, user: User, video_id: str) -> None:
|
||||
"""The video back to pristine for this user: status, resume point and `watched_at` all go.
|
||||
Playlist membership lives elsewhere and stays."""
|
||||
row = queries.video_state(db, user, video_id)
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
|
||||
|
||||
def set_progress(db: Session, user: User, video_id: str, position: int) -> None:
|
||||
"""The player's resume point. 0 clears it: a row holding nothing else goes, one with a status
|
||||
(watched, hidden) keeps it with the position zeroed. The status itself is never changed here."""
|
||||
row = queries.video_state(db, user, video_id)
|
||||
now = datetime.now(timezone.utc)
|
||||
if not position:
|
||||
if row is None:
|
||||
return
|
||||
if row.status == "new":
|
||||
db.delete(row)
|
||||
else:
|
||||
row.position_seconds = 0
|
||||
row.progress_updated_at = now
|
||||
db.flush()
|
||||
return
|
||||
if row is None:
|
||||
row = VideoState(user_id=user.id, video_id=video_id)
|
||||
db.add(row)
|
||||
row.position_seconds = position
|
||||
row.progress_updated_at = now
|
||||
db.flush()
|
||||
|
||||
|
||||
# --- playlists -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MoveRefused(Exception):
|
||||
"""A move between playlists that cannot happen: `reason` is the code the UI translates."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def touch(pl: Playlist) -> None:
|
||||
"""Bump the playlist's `updated_at` for an edit that only writes `playlist_items`.
|
||||
|
||||
`UpdatedAtMixin`'s `onupdate` fires on an UPDATE of the PLAYLIST row, and add / remove /
|
||||
reorder / move write the item rows instead — so without this the rail's "recently changed"
|
||||
sort would only ever see renames. The assignment is what makes the column mean "last time
|
||||
this list changed", which is what the rail promises the reader.
|
||||
|
||||
The DATABASE's clock, like the mixin's own `onupdate`: one sort key written from two clocks
|
||||
(this process's and Postgres's) could order a rename and an item edit the wrong way round."""
|
||||
pl.updated_at = func.now()
|
||||
|
||||
|
||||
def _next_playlist_position(db: Session, user_id: int) -> int:
|
||||
"""Append position for a new playlist in this user's rail."""
|
||||
return (
|
||||
db.execute(
|
||||
select(func.coalesce(func.max(Playlist.position), -1)).where(
|
||||
owned_by(Playlist.user_id, user_id)
|
||||
)
|
||||
).scalar_one()
|
||||
+ 1
|
||||
)
|
||||
|
||||
|
||||
def next_item_position(db: Session, playlist_id: int) -> int:
|
||||
"""Append position for a new item at the end of a playlist. Locks the playlist row first: two
|
||||
adds (two tabs, the rail and the page) otherwise both read the same max and land on one
|
||||
position, and the list's order between them is then arbitrary."""
|
||||
lock_playlists(db, playlist_id)
|
||||
return (
|
||||
db.execute(
|
||||
select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
|
||||
PlaylistItem.playlist_id == playlist_id
|
||||
)
|
||||
).scalar_one()
|
||||
+ 1
|
||||
)
|
||||
|
||||
|
||||
def create_playlist(db: Session, user: User, name: str) -> Playlist:
|
||||
pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id))
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
return pl
|
||||
|
||||
|
||||
def watch_later(db: Session, user: User) -> Playlist:
|
||||
"""The user's built-in Watch later playlist, created on first use."""
|
||||
pl = queries.watch_later(db, user)
|
||||
if pl is None:
|
||||
pl = Playlist(
|
||||
user_id=user.id,
|
||||
name="Watch later",
|
||||
kind="watch_later",
|
||||
position=_next_playlist_position(db, user.id),
|
||||
)
|
||||
db.add(pl)
|
||||
db.flush() # the id, for the item the caller adds next
|
||||
return pl
|
||||
|
||||
|
||||
def add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Append a video to a playlist if not already present (idempotent). Returns whether it
|
||||
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later).
|
||||
Locked BEFORE the presence check: two adds of one video both saw "absent" and the second one's
|
||||
insert hit uq_playlist_video as a 500."""
|
||||
lock_playlists(db, pl.id)
|
||||
if queries.item(db, pl.id, video_id) is not None:
|
||||
return False
|
||||
db.add(
|
||||
PlaylistItem(playlist_id=pl.id, video_id=video_id, position=next_item_position(db, pl.id))
|
||||
)
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
touch(pl)
|
||||
db.flush()
|
||||
return True
|
||||
|
||||
|
||||
def remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Remove a video from a playlist if present (idempotent). Returns whether it was removed.
|
||||
`mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
||||
lock_playlists(db, pl.id)
|
||||
found = queries.item(db, pl.id, video_id)
|
||||
if found is None:
|
||||
return False
|
||||
db.delete(found)
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
touch(pl)
|
||||
db.flush()
|
||||
return True
|
||||
|
||||
|
||||
def place_item(db: Session, pl: Playlist, video_id: str, position: int) -> None:
|
||||
"""Put a video already in the list at `position` (clamped to the end), renumbering the whole
|
||||
list: positions can have gaps after removals, so "index N" is only meaningful over the
|
||||
ordered rows."""
|
||||
rows = queries.items(db, pl.id)
|
||||
new = next(r for r in rows if r.video_id == video_id)
|
||||
rows.remove(new)
|
||||
rows.insert(min(position, len(rows)), new)
|
||||
for pos, row in enumerate(rows):
|
||||
row.position = pos
|
||||
db.flush()
|
||||
|
||||
|
||||
def add_to_playlist(db: Session, pl: Playlist, video_id: str, position: int | None) -> None:
|
||||
"""Add a video to one of the user's lists, at `position` when given (the list page's "Undo" of
|
||||
a removal puts the row back where it was in ONE request), else at the end. A video already in
|
||||
the list stays where it is."""
|
||||
added = add_item(db, pl, video_id, mark_dirty=False)
|
||||
if added and position is not None:
|
||||
place_item(db, pl, video_id, position)
|
||||
recompute_dirty(db, pl)
|
||||
db.flush()
|
||||
|
||||
|
||||
def rename_playlist(db: Session, pl: Playlist, name: str) -> None:
|
||||
"""A new name moves `dirty`; the caller holds the playlist's lock (the scheduled mirror refresh
|
||||
reads `dirty` under it)."""
|
||||
if name != pl.name:
|
||||
pl.name = name
|
||||
recompute_dirty(db, pl)
|
||||
db.flush()
|
||||
|
||||
|
||||
def delete_playlist(db: Session, pl: Playlist) -> None:
|
||||
"""The playlist first, as every item write: the cascade deletes the items after the list row
|
||||
is locked."""
|
||||
lock_playlists(db, pl.id)
|
||||
db.delete(pl) # items cascade
|
||||
db.flush()
|
||||
|
||||
|
||||
def move_item(db: Session, src: Playlist, dst: Playlist, video_id: str) -> None:
|
||||
"""Move one video from `src` into `dst`, in the caller's one transaction.
|
||||
|
||||
The client-side shape — add to the target, then remove from the source — leaves the video in
|
||||
BOTH lists whenever the second call fails. Idempotent: a video already in the target is simply
|
||||
dropped from the source (moving into a list that has it is a removal, not a duplicate). Raises
|
||||
`MoveRefused` for the same list twice, or a video a concurrent edit already took out."""
|
||||
if src.id == dst.id:
|
||||
raise MoveRefused("same_playlist")
|
||||
# Both lists, in id order: a move A→B beside a move B→A would otherwise lock crosswise.
|
||||
lock_playlists(db, src.id, dst.id)
|
||||
row = queries.item(db, src.id, video_id)
|
||||
if row is None:
|
||||
raise MoveRefused("not_in_playlist")
|
||||
dst_changed = False
|
||||
if queries.item(db, dst.id, video_id) is None:
|
||||
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
|
||||
# survives the move, and there is no window where the video belongs to neither list. The
|
||||
# check above holds: both lists are locked, and an insert into the target (another tab's
|
||||
# add, a YouTube merge) waits for the lock — its foreign key needs the target row.
|
||||
row.playlist_id = dst.id
|
||||
row.position = next_item_position(db, dst.id)
|
||||
dst_changed = True
|
||||
if not dst_changed:
|
||||
db.delete(row)
|
||||
# The source always changed; the target only when the row landed in it. Touching a target
|
||||
# that already had the video promoted an untouched list in the rail's "recently changed" sort.
|
||||
recompute_dirty(db, src)
|
||||
touch(src)
|
||||
if dst_changed:
|
||||
recompute_dirty(db, dst)
|
||||
touch(dst)
|
||||
db.flush()
|
||||
|
||||
|
||||
def reorder_items(db: Session, pl: Playlist, order: list[str]) -> int:
|
||||
"""Number the listed videos 0..n in the listed order. Items the list left out (added in another
|
||||
tab meanwhile) keep their relative order after them, so no two items share a position. The
|
||||
caller holds the playlist's lock. Returns how many items the list has."""
|
||||
by_video = {it.video_id: it for it in queries.items(db, pl.id)}
|
||||
pos = 0
|
||||
seen: set[str] = set()
|
||||
for vid in order:
|
||||
it = by_video.get(vid)
|
||||
if it is not None:
|
||||
it.position = pos
|
||||
pos += 1
|
||||
seen.add(vid)
|
||||
for it in sorted(
|
||||
(it for vid, it in by_video.items() if vid not in seen), key=lambda i: i.position
|
||||
):
|
||||
it.position = pos
|
||||
pos += 1
|
||||
recompute_dirty(db, pl)
|
||||
touch(pl)
|
||||
db.flush()
|
||||
return pos
|
||||
@@ -0,0 +1,17 @@
|
||||
"""The account filter the activity page's lists share (runs, admin actions, server log, quota): one
|
||||
filter bar drives them all, so the value it sends means the same on every tab.
|
||||
|
||||
The value is an account id, or `none` — the rows no account made (the background) — a value to filter
|
||||
on, not the absence of a filter.
|
||||
"""
|
||||
|
||||
from sqlalchemy import ColumnElement
|
||||
|
||||
NONE = "none"
|
||||
# The `user` query parameter's shape on every route that takes it.
|
||||
PATTERN = r"^(none|[0-9]{1,10})$"
|
||||
|
||||
|
||||
def condition(column, value: str) -> ColumnElement[bool]:
|
||||
"""`column` (an account id column) narrowed to `value`."""
|
||||
return column.is_(None) if value == NONE else column == int(value)
|
||||
@@ -16,6 +16,7 @@ from app.db import Base
|
||||
from app.models import POSITION_MAX_S, User
|
||||
from app.issues import gitea
|
||||
from app.refusal import refused
|
||||
from app.userscope import owned_row
|
||||
from app.youtube.client import YouTubeError, refusal_reason
|
||||
|
||||
M = TypeVar("M", bound=Base)
|
||||
@@ -90,8 +91,8 @@ def owned_or_404(
|
||||
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
||||
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
|
||||
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
|
||||
obj = get_or_404(db, model, id_, what=what, lock=lock)
|
||||
if getattr(obj, owner_attr) != user.id:
|
||||
obj = owned_row(db, model, id_, user, owner_attr=owner_attr, lock=lock)
|
||||
if obj is None:
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Keyset paging over `(timestamp desc, id desc)`, shared by the admin history lists (server log,
|
||||
audit log, background runs).
|
||||
|
||||
A timestamp alone is not unique (a browser log row's `ts` is clamped into its consent window; a run
|
||||
and its siblings can start in the same microsecond), and an offset would skip or repeat rows while
|
||||
writers add new ones at the top — so a page continues from the last row's `(timestamp, id)`.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import and_, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import User
|
||||
|
||||
PAGE_MAX = 200
|
||||
|
||||
|
||||
def utc(value: datetime | None) -> datetime | None:
|
||||
"""A bound without a zone is UTC — the columns are `timestamptz`, and a naive value would be read
|
||||
in the database session's zone instead."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def encode_cursor(ts: datetime, row_id: int) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps([ts.isoformat(), row_id]).encode()).decode()
|
||||
|
||||
|
||||
def decode_cursor(cursor: str) -> tuple[datetime, int]:
|
||||
try:
|
||||
ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
at = datetime.fromisoformat(ts)
|
||||
return (at if at.tzinfo is not None else at.replace(tzinfo=timezone.utc)), int(row_id)
|
||||
except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc:
|
||||
raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc
|
||||
|
||||
|
||||
def older_than(ts_col, id_col, cursor: str):
|
||||
"""The rows after `cursor` in `(ts desc, id desc)` order. The `ts <=` conjunct is implied by the
|
||||
OR; it is there so the planner can bound the timestamp index scan instead of filtering every
|
||||
newer row first."""
|
||||
ts, row_id = decode_cursor(cursor)
|
||||
return and_(ts_col <= ts, or_(ts_col < ts, and_(ts_col == ts, id_col < row_id)))
|
||||
|
||||
|
||||
def page(rows: list, limit: int, ts_of) -> tuple[list, str | None]:
|
||||
"""`rows` was read with `limit + 1` (to know whether there is a next page without a count query):
|
||||
the page itself, and the cursor that continues after it (None on the last page)."""
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return rows, encode_cursor(ts_of(rows[-1]), rows[-1].id) if more else None
|
||||
|
||||
|
||||
def accounts(db: Session, user_ids: set[int]) -> dict[str, str]:
|
||||
"""The accounts a page names, by id → address. An id with no account left (erased after the row
|
||||
was written) is simply absent, and the client shows the bare id."""
|
||||
if not user_ids:
|
||||
return {}
|
||||
rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all()
|
||||
return {str(uid): email for uid, email in rows}
|
||||
@@ -8,33 +8,29 @@ What a row holds was redacted when it was written (`logs.record_fields`, and the
|
||||
browser's rows), so this serves the columns as they are; a value the redactor took out reads as
|
||||
`[redacted]` / `[email]`, which is the point — the console shows that something was there.
|
||||
|
||||
Paging is a keyset over `(ts desc, id desc)`: a browser row's `ts` is clamped into its consent
|
||||
window, so `ts` alone is not unique, and an offset would skip or repeat rows while the writer adds
|
||||
new ones at the top. The live tail re-reads the first page rather than asking for "ids after N":
|
||||
Paging is a keyset over `(ts desc, id desc)` (`_keyset`). The live tail re-reads the first page rather than asking for "ids after N":
|
||||
several processes write this table, and a batch's ids are taken when it inserts, not when it
|
||||
commits, so a slower batch can land below an id the tail already passed — and would never show.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import Text, and_, cast, func, or_, select
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import Text, cast, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logsink
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import LogEvent, User
|
||||
from app.routes import _account as account
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.utils import like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/logs", tags=["admin"])
|
||||
|
||||
PAGE_MAX = 200
|
||||
# A facet list is a chip row, not a report: past this many distinct events the rest are reachable
|
||||
# through the free-text search, and the response stays small whatever a self-host logs.
|
||||
FACET_EVENTS_MAX = 100
|
||||
@@ -42,38 +38,9 @@ FACET_EVENTS_MAX = 100
|
||||
Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
# The filters a facet can leave out of its own count (`LogFilters.conditions`).
|
||||
Facet = Literal["levels", "sources", "user", "event"]
|
||||
# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous
|
||||
# request) — a real value to filter on, not the absence of a filter.
|
||||
NONE = "none"
|
||||
|
||||
|
||||
def _utc(value: datetime | None) -> datetime | None:
|
||||
"""A bound without a zone is UTC — the column is `timestamptz`, and a naive value would be read
|
||||
in the database session's zone instead."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _encode_cursor(row: LogEvent) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps([row.ts.isoformat(), row.id]).encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(cursor: str) -> tuple[datetime, int]:
|
||||
try:
|
||||
ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
return _utc(datetime.fromisoformat(ts)), int(row_id)
|
||||
except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc:
|
||||
raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc
|
||||
|
||||
|
||||
def _accounts(db: Session, user_ids: set[int]) -> dict[str, str]:
|
||||
"""The accounts a page names, by id → address. An id with no account left (erased after the
|
||||
line was written) is simply absent, and the client shows the bare id."""
|
||||
if not user_ids:
|
||||
return {}
|
||||
rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all()
|
||||
return {str(uid): email for uid, email in rows}
|
||||
# `none` selects the rows that have no event code, as the account filter's `none` (app/routes/_account)
|
||||
# selects the rows of no account — a real value to filter on, not the absence of a filter.
|
||||
NONE = account.NONE
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -86,6 +53,7 @@ class LogFilters:
|
||||
event: str | None
|
||||
request_id: str | None
|
||||
job_id: int | None
|
||||
run_id: int | None
|
||||
q: str | None
|
||||
since: datetime | None
|
||||
until: datetime | None
|
||||
@@ -100,11 +68,7 @@ class LogFilters:
|
||||
if self.sources and skip != "sources":
|
||||
out.append(LogEvent.source.in_(self.sources))
|
||||
if self.user is not None and skip != "user":
|
||||
out.append(
|
||||
LogEvent.user_id.is_(None)
|
||||
if self.user == NONE
|
||||
else LogEvent.user_id == int(self.user)
|
||||
)
|
||||
out.append(account.condition(LogEvent.user_id, self.user))
|
||||
if self.event is not None and skip != "event":
|
||||
out.append(
|
||||
LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event
|
||||
@@ -113,10 +77,12 @@ class LogFilters:
|
||||
out.append(LogEvent.request_id == self.request_id)
|
||||
if self.job_id is not None:
|
||||
out.append(LogEvent.job_id == self.job_id)
|
||||
if self.run_id is not None:
|
||||
out.append(LogEvent.run_id == self.run_id)
|
||||
if self.since is not None:
|
||||
out.append(LogEvent.ts >= _utc(self.since))
|
||||
out.append(LogEvent.ts >= utc(self.since))
|
||||
if self.until is not None:
|
||||
out.append(LogEvent.ts < _utc(self.until))
|
||||
out.append(LogEvent.ts < utc(self.until))
|
||||
if self.q and self.q.strip():
|
||||
like = f"%{like_escape(self.q.strip())}%"
|
||||
out.append(
|
||||
@@ -134,17 +100,20 @@ class LogFilters:
|
||||
def log_filters(
|
||||
levels: list[Level] = Query(default=[]),
|
||||
sources: list[str] = Query(default=[], max_length=8),
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
user: str | None = Query(default=None, pattern=account.PATTERN),
|
||||
event: str | None = Query(
|
||||
default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$"
|
||||
),
|
||||
request_id: str | None = Query(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$"),
|
||||
job_id: int | None = Query(default=None, ge=1),
|
||||
run_id: int | None = Query(default=None, ge=1),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> LogFilters:
|
||||
return LogFilters(list(levels), list(sources), user, event, request_id, job_id, q, since, until)
|
||||
return LogFilters(
|
||||
list(levels), list(sources), user, event, request_id, job_id, run_id, q, since, until
|
||||
)
|
||||
|
||||
|
||||
@router.get("")
|
||||
@@ -159,28 +128,20 @@ def list_logs(
|
||||
continues from a previous page's `next_cursor`."""
|
||||
query = select(LogEvent).where(*filters.conditions())
|
||||
if before is not None:
|
||||
ts, row_id = _decode_cursor(before)
|
||||
# The `ts <=` conjunct is implied by the OR; it is there so the planner can bound the
|
||||
# `ts` index scan instead of filtering every newer row first.
|
||||
query = query.where(
|
||||
and_(
|
||||
LogEvent.ts <= ts,
|
||||
or_(LogEvent.ts < ts, and_(LogEvent.ts == ts, LogEvent.id < row_id)),
|
||||
)
|
||||
)
|
||||
|
||||
# One more than asked, to know whether there is a next page without a second count query.
|
||||
rows = (
|
||||
db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1))
|
||||
.scalars()
|
||||
.all()
|
||||
query = query.where(older_than(LogEvent.ts, LogEvent.id, before))
|
||||
rows, cursor = page(
|
||||
list(
|
||||
db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1))
|
||||
.scalars()
|
||||
.all()
|
||||
),
|
||||
limit,
|
||||
lambda r: r.ts,
|
||||
)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return {
|
||||
"items": [logsink.serialize(r) for r in rows],
|
||||
"accounts": _accounts(db, {r.user_id for r in rows if r.user_id is not None}),
|
||||
"next_cursor": _encode_cursor(rows[-1]) if more else None,
|
||||
"accounts": accounts(db, {r.user_id for r in rows if r.user_id is not None}),
|
||||
"next_cursor": cursor,
|
||||
}
|
||||
|
||||
|
||||
@@ -211,6 +172,6 @@ def log_facets(
|
||||
"sources": counts(LogEvent.source, "sources"),
|
||||
"events": counts(LogEvent.event, "event", FACET_EVENTS_MAX),
|
||||
"users": [{**u, "value": str(u["value"])} for u in users],
|
||||
"accounts": _accounts(db, {u["value"] for u in users if u["value"] != NONE}),
|
||||
"accounts": accounts(db, {u["value"] for u in users if u["value"] != NONE}),
|
||||
"total": db.scalar(total) or 0,
|
||||
}
|
||||
|
||||
+106
-29
@@ -1,67 +1,144 @@
|
||||
"""Admin audit log API: read the append-only who/what/when trail, and clear it.
|
||||
|
||||
Admin-only. The client renders these rows in a DataTable (client-side sort/filter/paginate), so
|
||||
this returns the most recent window of rows (capped) plus the true total, rather than paging
|
||||
server-side — the log stays small in practice (state-changing runs + admin actions only)."""
|
||||
Admin-only. The activity page reads it a page at a time, newest first, filtered on the server like
|
||||
the server log (`_keyset` paging), so the whole trail is reachable however long it grows."""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import delete, func, select
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import Text, cast, delete, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import AuditLog, User
|
||||
from app.utils import iso
|
||||
from app.models import AuditLog, LogEvent, User
|
||||
from app.routes import _account as account
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.utils import iso, like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/audit", tags=["admin"])
|
||||
|
||||
# Cap the client-side window. The audit log is small by design; if it ever outgrows this we'd
|
||||
# switch the DataTable to server-side paging.
|
||||
MAX_ROWS = 2000
|
||||
|
||||
|
||||
def _serialize(row: AuditLog, actors: dict[int, str]) -> dict:
|
||||
def _serialize(row: AuditLog, lines: dict[str, int]) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
"created_at": iso(row.created_at),
|
||||
# actor_id NULL = the scheduler/background did it (client shows "System"); a set id that
|
||||
# can't be resolved would mean a deleted user, but SET NULL makes that NULL — so present.
|
||||
"actor": actors.get(row.actor_id) if row.actor_id is not None else None,
|
||||
# NULL = the background did it (the client shows "System"); the address is in `accounts`.
|
||||
"actor_id": row.actor_id,
|
||||
"action": row.action,
|
||||
"target_type": row.target_type,
|
||||
"target_id": row.target_id,
|
||||
"summary": row.summary,
|
||||
"before": row.before,
|
||||
"after": row.after,
|
||||
"request_id": row.request_id,
|
||||
# How many of that request's log lines are still kept: they go after days, the row after
|
||||
# months, and a link to none would open an empty list.
|
||||
"log_lines": lines.get(row.request_id, 0) if row.request_id else 0,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AuditFilters:
|
||||
action: str | None
|
||||
actor: str | None
|
||||
q: str | None
|
||||
since: datetime | None
|
||||
until: datetime | None
|
||||
|
||||
def conditions(self, skip: Literal["action"] | None = None) -> list:
|
||||
"""The WHERE conditions, ANDed. `skip="action"` leaves the action filter out, so the action
|
||||
menu counts every action under the other filters (the way back from a pick)."""
|
||||
out = []
|
||||
if self.action is not None and skip != "action":
|
||||
out.append(AuditLog.action == self.action)
|
||||
if self.actor is not None:
|
||||
out.append(account.condition(AuditLog.actor_id, self.actor))
|
||||
if self.since is not None:
|
||||
out.append(AuditLog.created_at >= utc(self.since))
|
||||
if self.until is not None:
|
||||
out.append(AuditLog.created_at < utc(self.until))
|
||||
if self.q and self.q.strip():
|
||||
like = f"%{like_escape(self.q.strip())}%"
|
||||
out.append(
|
||||
or_(
|
||||
AuditLog.summary.ilike(like),
|
||||
AuditLog.target_id.ilike(like),
|
||||
AuditLog.action.ilike(like),
|
||||
AuditLog.request_id.ilike(like),
|
||||
cast(AuditLog.before, Text).ilike(like),
|
||||
cast(AuditLog.after, Text).ilike(like),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def audit_filters(
|
||||
action: str | None = Query(default=None, max_length=48, pattern=r"^[a-z][a-z0-9_]*$"),
|
||||
user: str | None = Query(default=None, pattern=account.PATTERN),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> AuditFilters:
|
||||
return AuditFilters(action, user, q, since, until)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_audit(
|
||||
limit: int = 500,
|
||||
filters: AuditFilters = Depends(audit_filters),
|
||||
before: str | None = Query(default=None, max_length=200),
|
||||
limit: int = Query(default=100, ge=1, le=PAGE_MAX),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
limit = max(1, min(limit, MAX_ROWS))
|
||||
total = db.scalar(select(func.count()).select_from(AuditLog)) or 0
|
||||
rows = (
|
||||
db.execute(select(AuditLog).order_by(AuditLog.created_at.desc()).limit(limit))
|
||||
.scalars()
|
||||
.all()
|
||||
"""One page of rows, newest first, matching every filter given; `before` continues from a
|
||||
previous page's `next_cursor`."""
|
||||
query = select(AuditLog).where(*filters.conditions())
|
||||
if before is not None:
|
||||
query = query.where(older_than(AuditLog.created_at, AuditLog.id, before))
|
||||
order = (AuditLog.created_at.desc(), AuditLog.id.desc())
|
||||
rows, cursor = page(
|
||||
list(db.execute(query.order_by(*order).limit(limit + 1)).scalars().all()),
|
||||
limit,
|
||||
lambda r: r.created_at,
|
||||
)
|
||||
actor_ids = {r.actor_id for r in rows if r.actor_id is not None}
|
||||
actors: dict[int, str] = {}
|
||||
if actor_ids:
|
||||
actors = dict(db.execute(select(User.id, User.email).where(User.id.in_(actor_ids))).all())
|
||||
requests = {r.request_id for r in rows if r.request_id}
|
||||
lines: dict[str, int] = {}
|
||||
if requests:
|
||||
counted = db.execute(
|
||||
select(LogEvent.request_id, func.count())
|
||||
.where(LogEvent.request_id.in_(requests))
|
||||
.group_by(LogEvent.request_id)
|
||||
)
|
||||
lines = {rid: n for rid, n in counted.tuples() if rid is not None}
|
||||
return {
|
||||
"items": [_serialize(r, actors) for r in rows],
|
||||
"total": total,
|
||||
"returned": len(rows),
|
||||
"items": [_serialize(r, lines) for r in rows],
|
||||
"accounts": accounts(db, {r.actor_id for r in rows if r.actor_id is not None}),
|
||||
"next_cursor": cursor,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/facets")
|
||||
def audit_facets(
|
||||
filters: AuditFilters = Depends(audit_filters),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The actions present under the other filters, with counts — what the action menu offers."""
|
||||
n = func.count().label("n")
|
||||
stmt = (
|
||||
select(AuditLog.action, n)
|
||||
.where(*filters.conditions("action"))
|
||||
.group_by(AuditLog.action)
|
||||
.order_by(n.desc(), AuditLog.action)
|
||||
)
|
||||
return {"actions": [{"value": a, "count": c} for a, c in db.execute(stmt).all()]}
|
||||
|
||||
|
||||
@router.delete("")
|
||||
def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Clear the audit log. Leaves ONE row recording who cleared it + how many (tamper-evidence)."""
|
||||
|
||||
@@ -3,30 +3,28 @@ hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side w
|
||||
(subscribe / unsubscribe) that require the optional write scope."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from functools import partial
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import Select, func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.collation import collated, sorted_words
|
||||
from app.mailtext import lang_for_user
|
||||
from app import logs, quota
|
||||
from app import logs, quota, scheduler
|
||||
from app.auth import admin_user, current_user, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
ChannelTag,
|
||||
ExploredChannel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
Subscription,
|
||||
Tag,
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, Tag
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.sync.explore import explore_ingest_page
|
||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||
@@ -481,7 +479,6 @@ class ChannelUpdateIn(BaseModel):
|
||||
def update_channel(
|
||||
channel_id: str,
|
||||
body: ChannelUpdateIn,
|
||||
background: BackgroundTasks,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -508,12 +505,19 @@ def update_channel(
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
||||
run_recent_backfill(db, [channel], max_channels=1)
|
||||
|
||||
# …and kick the FULL deep backfill for this one channel immediately in the background, so the
|
||||
# whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. The
|
||||
# request returns right away; the channel page's live poll shows the count climb. The scheduler
|
||||
# still finishes a very large channel's tail on its normal runs (deep_requested stays set).
|
||||
if deep_turned_on:
|
||||
background.add_task(backfill_channel_now, channel_id, user.id)
|
||||
# …and kick the FULL deep backfill for this one channel immediately, as a recorded run of its own,
|
||||
# so the whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks.
|
||||
# The request returns right away; the channel page's live poll shows the count climb. The
|
||||
# scheduler still finishes a very large channel's tail on its normal runs (deep_requested stays set).
|
||||
# A channel whose history is all in gets no run: it would only say "nothing to do", one kept row
|
||||
# per flip of the flag.
|
||||
if deep_turned_on and not (channel is not None and channel.backfill_done):
|
||||
scheduler.run_ad_hoc(
|
||||
scheduler.CHANNEL_BACKFILL,
|
||||
partial(backfill_channel_now, channel_id, user.id),
|
||||
actor_id=user.id,
|
||||
target=channel_id,
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
|
||||
@@ -61,7 +61,7 @@ def set_config(
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
s = sysconfig.config_page_spec(key)
|
||||
if s is None:
|
||||
raise not_found("config_key")
|
||||
if "value" not in body.model_fields_set:
|
||||
@@ -89,7 +89,9 @@ def set_config(
|
||||
before=None if s.secret else {"value": _redact(old)},
|
||||
after=None if s.secret else {"value": _redact(new)},
|
||||
)
|
||||
db.commit()
|
||||
# The override is committed even when the trail skips it: re-saving the value in force still
|
||||
# pins it as an override, and an uncommitted write would be dropped with the session.
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
@@ -100,7 +102,7 @@ def reset_config(
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
s = sysconfig.config_page_spec(key)
|
||||
if s is None:
|
||||
raise not_found("config_key")
|
||||
old = None if s.secret else sysconfig.get(db, key)
|
||||
|
||||
@@ -13,7 +13,6 @@ from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -22,9 +21,9 @@ from app.db import get_db
|
||||
from app.discover import hero as hero_mod
|
||||
from app.discover import pins, rails
|
||||
from app.discover.budget import Budget, DEFAULT_BUDGET
|
||||
from app.models import ComfortPin, User
|
||||
from app.models import User
|
||||
from app.plex.cards import norm_lang
|
||||
from app.routes._common import owned_or_404, not_found
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/discover", tags=["discover"])
|
||||
|
||||
@@ -140,30 +139,19 @@ def add_pin(
|
||||
raise not_found("item")
|
||||
column, value = resolved
|
||||
|
||||
existing = db.execute(
|
||||
select(ComfortPin).where(
|
||||
ComfortPin.user_id == user.id,
|
||||
getattr(ComfortPin, column) == value,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
existing = pins.pin_of(db, user, column, value)
|
||||
if existing is not None:
|
||||
return {"ok": True, "id": existing.id, "created": False}
|
||||
|
||||
pin = ComfortPin(user_id=user.id, position=pins.next_position(db, user), **{column: value})
|
||||
db.add(pin)
|
||||
try:
|
||||
pin = pins.add(db, user, column, value)
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
# A concurrent pin of the same thing from a second tab. The unique constraint did its job;
|
||||
# the user's intent ("be on my shelf") is satisfied either way, so report the winner rather
|
||||
# than a 500 the user can do nothing with.
|
||||
db.rollback()
|
||||
winner = db.execute(
|
||||
select(ComfortPin).where(
|
||||
ComfortPin.user_id == user.id,
|
||||
getattr(ComfortPin, column) == value,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
winner = pins.pin_of(db, user, column, value)
|
||||
if winner is None:
|
||||
# Not the unique constraint, then: the only other way in is the target row being pruned
|
||||
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
||||
@@ -180,8 +168,10 @@ def remove_pin(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pin = owned_or_404(db, ComfortPin, pin_id, user, what="pin")
|
||||
db.delete(pin)
|
||||
pin = pins.own_pin(db, user, pin_id)
|
||||
if pin is None:
|
||||
raise not_found("pin")
|
||||
pins.remove(db, pin)
|
||||
db.commit()
|
||||
return {"deleted": pin_id}
|
||||
|
||||
|
||||
+106
-337
@@ -3,23 +3,22 @@
|
||||
All routes are behind `current_user`; admin routes add `admin_user`. Live job state is polled by the
|
||||
frontend (the worker is a separate process); this module never downloads — it enqueues, manages job
|
||||
state, and shares finished files. The library itself — listing, renaming, moving, deleting and
|
||||
serving a done file, and uploading one — is `routes.files`.
|
||||
serving a done file, and uploading one — is `routes.files`. Reads come from `downloads.queries`,
|
||||
writes go through Capture's service (`service`, `links`, `quota`); this module validates, answers in
|
||||
HTTP and commits.
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Literal
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated
|
||||
from app import audit, sysconfig
|
||||
from app.auth import admin_user, current_user
|
||||
from app.config import settings
|
||||
@@ -29,45 +28,32 @@ from app.downloads import credentials as dlcreds
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import folders
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads import queries, quota, service, storage
|
||||
from app.downloads import urlguard
|
||||
from app.downloads.folders import FolderKind
|
||||
from app.downloads.status import FINISHED, NOT_REFETCHABLE, JobStatus
|
||||
from app.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
FileFolder,
|
||||
DownloadProfile,
|
||||
DownloadQuota,
|
||||
DownloadShare,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
User,
|
||||
display_text,
|
||||
)
|
||||
from app.downloads.status import NOT_REFETCHABLE, JobStatus
|
||||
from app.models import User, display_text
|
||||
from app.routes._common import (
|
||||
ProgressIn,
|
||||
get_or_404,
|
||||
kept_position,
|
||||
not_found,
|
||||
owned_or_404,
|
||||
)
|
||||
from app.routes.files import (
|
||||
MAX_LIST,
|
||||
accessible_job,
|
||||
assets_for,
|
||||
own_folder,
|
||||
own_job,
|
||||
own_job_for_update,
|
||||
positions_for,
|
||||
quota_detail,
|
||||
serialize,
|
||||
serialize_job,
|
||||
source_keys,
|
||||
)
|
||||
from app.security import hash_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.userscope import emails_of, is_messageable_user, messageable_others, user_by_email
|
||||
from app.refusal import refused
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.downloads.models import DownloadJob, DownloadLink, DownloadProfile, FileFolder
|
||||
|
||||
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
|
||||
admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"])
|
||||
|
||||
@@ -119,7 +105,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
# --- profiles ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _serialize_profile(p: DownloadProfile) -> dict:
|
||||
def _serialize_profile(p: "DownloadProfile") -> dict:
|
||||
return {
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
@@ -131,18 +117,7 @@ def _serialize_profile(p: DownloadProfile) -> dict:
|
||||
|
||||
@router.get("/profiles")
|
||||
def list_profiles(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
rows = (
|
||||
db.execute(
|
||||
select(DownloadProfile)
|
||||
.where((DownloadProfile.is_builtin.is_(True)) | (DownloadProfile.user_id == user.id))
|
||||
.order_by(
|
||||
DownloadProfile.is_builtin.desc(), DownloadProfile.sort_order, DownloadProfile.id
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return [_serialize_profile(p) for p in rows]
|
||||
return [_serialize_profile(p) for p in queries.profiles_for(db, user)]
|
||||
|
||||
|
||||
class CreateProfileIn(BaseModel):
|
||||
@@ -158,15 +133,7 @@ def create_profile(
|
||||
spec = body.spec
|
||||
if not name:
|
||||
raise refused(400, "profile_name_required")
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
|
||||
DownloadProfile.user_id == user.id
|
||||
)
|
||||
).scalar_one()
|
||||
p = DownloadProfile(
|
||||
user_id=user.id, name=name[:80], spec=spec, is_builtin=False, sort_order=maxpos + 1
|
||||
)
|
||||
db.add(p)
|
||||
p = service.create_profile(db, user, name, spec)
|
||||
db.commit()
|
||||
return _serialize_profile(p)
|
||||
|
||||
@@ -183,31 +150,32 @@ def update_profile(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise not_found("profile")
|
||||
p = _own_profile(db, user, profile_id)
|
||||
sent = body.model_fields_set
|
||||
name = None
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise refused(400, "profile_name_required")
|
||||
p.name = name[:80]
|
||||
if "spec" in sent:
|
||||
if body.spec is None:
|
||||
raise refused(400, "spec_not_object")
|
||||
p.spec = body.spec
|
||||
if "spec" in sent and body.spec is None:
|
||||
raise refused(400, "spec_not_object")
|
||||
service.update_profile(p, name=name, spec=body.spec if "spec" in sent else None)
|
||||
db.commit()
|
||||
return _serialize_profile(p)
|
||||
|
||||
|
||||
def _own_profile(db: Session, user: User, profile_id: int) -> "DownloadProfile":
|
||||
p = queries.own_profile(db, user, profile_id)
|
||||
if p is None:
|
||||
raise not_found("profile")
|
||||
return p
|
||||
|
||||
|
||||
@router.delete("/profiles/{profile_id}")
|
||||
def delete_profile(
|
||||
profile_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise not_found("profile")
|
||||
db.delete(p)
|
||||
service.delete_profile(db, _own_profile(db, user, profile_id))
|
||||
db.commit()
|
||||
return {"deleted": profile_id}
|
||||
|
||||
@@ -277,18 +245,13 @@ def _resolve_spec(
|
||||
"""Pick the format spec for an enqueue: an explicit profile_id (builtin or the user's own),
|
||||
an inline spec, or the first builtin as a fallback."""
|
||||
if profile_id is not None:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or (not p.is_builtin and p.user_id != user.id):
|
||||
p = queries.usable_profile(db, user, profile_id)
|
||||
if p is None:
|
||||
raise not_found("profile")
|
||||
return p.spec, p.id
|
||||
if spec is not None:
|
||||
return spec, None
|
||||
fallback = db.execute(
|
||||
select(DownloadProfile)
|
||||
.where(DownloadProfile.is_builtin.is_(True))
|
||||
.order_by(DownloadProfile.sort_order)
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
fallback = queries.default_profile(db)
|
||||
if fallback is None:
|
||||
raise refused(400, "no_profile")
|
||||
return fallback.spec, fallback.id
|
||||
@@ -326,6 +289,7 @@ def enqueue_download(
|
||||
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -355,6 +319,7 @@ def enqueue_edit(
|
||||
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
|
||||
except service.EditError as e:
|
||||
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason}) from e
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -398,38 +363,22 @@ def enqueue_clip(
|
||||
raise HTTPException(status_code=status, detail=detail) from e
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
# How long a finished job stays in the queue: the tray polls it from every page and must see a job
|
||||
# turn done (its "Done · Open" row and the done notice) — the finished file itself is the library's.
|
||||
# A done job is timed from when it became done, never from its last write: a rename, a move or the
|
||||
# GC's expiry warning touches an old file's row, and the tray would announce it as just finished.
|
||||
QUEUE_RECENT = timedelta(minutes=10)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_downloads(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
"""The queue: what is waiting, running, paused or failed, and what finished a moment ago."""
|
||||
recent = datetime.now(timezone.utc) - QUEUE_RECENT
|
||||
jobs = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.where(
|
||||
DownloadJob.user_id == user.id,
|
||||
DownloadJob.status.not_in(FINISHED)
|
||||
| ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent))
|
||||
| ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)),
|
||||
)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(MAX_LIST)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assets = assets_for(db, jobs)
|
||||
positions = positions_for(db, user, jobs)
|
||||
sources = source_keys(db, jobs)
|
||||
jobs = queries.queue(db, user, datetime.now(timezone.utc) - QUEUE_RECENT)
|
||||
assets = queries.assets_for(db, jobs)
|
||||
positions = queries.positions_for(db, user, jobs)
|
||||
sources = queries.source_keys(db, jobs)
|
||||
return [
|
||||
{
|
||||
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
|
||||
@@ -453,14 +402,8 @@ _INDEX_RANK: dict[str, int] = {
|
||||
def my_download_index(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""source_ref -> best active status, for the feed's per-card "downloaded / queued" badge.
|
||||
Small + cheap so the feed can poll it without loading the full job list."""
|
||||
rows = db.execute(
|
||||
select(DownloadJob.source_ref, DownloadJob.status).where(
|
||||
DownloadJob.user_id == user.id,
|
||||
DownloadJob.status.in_(tuple(_INDEX_RANK)),
|
||||
)
|
||||
).all()
|
||||
out: dict[str, str] = {}
|
||||
for ref, status in rows:
|
||||
for ref, status in queries.statuses_by_source(db, user, tuple(_INDEX_RANK)):
|
||||
if _INDEX_RANK.get(status, 0) > _INDEX_RANK.get(out.get(ref, ""), 0):
|
||||
out[ref] = status
|
||||
return out
|
||||
@@ -472,6 +415,7 @@ def pause_download(
|
||||
) -> dict:
|
||||
job = own_job_for_update(db, user, job_id)
|
||||
service.pause(db, job)
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -481,6 +425,7 @@ def resume_download(
|
||||
) -> dict:
|
||||
job = own_job_for_update(db, user, job_id)
|
||||
service.resume(db, job)
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -508,6 +453,7 @@ def redownload_download(
|
||||
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED):
|
||||
raise refused(409, "still_downloading")
|
||||
service.redownload(db, job)
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -517,6 +463,7 @@ def cancel_download(
|
||||
) -> dict:
|
||||
job = own_job_for_update(db, user, job_id)
|
||||
service.cancel(db, job)
|
||||
db.commit()
|
||||
return serialize_job(db, job)
|
||||
|
||||
|
||||
@@ -534,24 +481,7 @@ def set_library_progress(
|
||||
one shared with them). A position too early or too close to the end clears it instead."""
|
||||
accessible_job(db, user, job_id)
|
||||
position = kept_position(body)
|
||||
mine = (LibraryState.user_id == user.id) & (LibraryState.job_id == job_id)
|
||||
if not position:
|
||||
# A statement, not a loaded row's delete: the periodic checkpoint and the pagehide beacon of
|
||||
# one stop may both clear it, and the second then deletes nothing rather than failing.
|
||||
db.execute(delete(LibraryState).where(mine))
|
||||
db.commit()
|
||||
return {"ok": True, "job_id": job_id, "position_seconds": 0}
|
||||
now = datetime.now(timezone.utc)
|
||||
# One row per (user, job) even when two checkpoints race (the same pair): an upsert, not a
|
||||
# read-then-insert that the second would fail on the unique key.
|
||||
db.execute(
|
||||
pg_insert(LibraryState)
|
||||
.values(user_id=user.id, job_id=job_id, position_seconds=position, progress_updated_at=now)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_library_user_job",
|
||||
set_={"position_seconds": position, "progress_updated_at": now},
|
||||
)
|
||||
)
|
||||
service.set_position(db, user, job_id, position)
|
||||
db.commit()
|
||||
return {"ok": True, "job_id": job_id, "position_seconds": position}
|
||||
|
||||
@@ -564,8 +494,7 @@ def storyboard_meta(
|
||||
|
||||
Best-effort: for a very long/high-res source the sprite may not generate in time — then
|
||||
`available` is False and the editor shows a plain timeline."""
|
||||
job = accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
asset = queries.asset_of(db, accessible_job(db, user, job_id))
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise refused(409, "download_not_ready")
|
||||
meta = editmod.ensure_storyboard(asset, settings.files_root)
|
||||
@@ -585,8 +514,7 @@ def storyboard_meta(
|
||||
@router.get("/{job_id}/poster.jpg")
|
||||
def poster_image(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
|
||||
"""The locally-generated poster frame for a download whose source had no thumbnail."""
|
||||
job = accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
asset = queries.asset_of(db, accessible_job(db, user, job_id))
|
||||
if asset is None or not asset.poster_path:
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||
@@ -599,8 +527,7 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session =
|
||||
def storyboard_image(
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
):
|
||||
job = accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
asset = queries.asset_of(db, accessible_job(db, user, job_id))
|
||||
if asset is None or not asset.storyboard_path:
|
||||
raise not_found("filmstrip")
|
||||
path = storage.safe_abs_path(settings.files_root, asset.storyboard_path)
|
||||
@@ -623,45 +550,20 @@ def share_download(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
job = own_job(db, user, job_id)
|
||||
if job.status != JobStatus.DONE or job.asset_id is None:
|
||||
raise refused(409, "share_unfinished")
|
||||
target = body.email.strip().lower()
|
||||
if not target:
|
||||
raise refused(400, "recipient_required")
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == target)
|
||||
).scalar_one_or_none()
|
||||
recipient = user_by_email(db, target)
|
||||
# Uniform "no user" for a missing OR non-messageable (suspended/deactivated) target, so
|
||||
# sharing can't be used to probe which addresses exist or to reach a suspended account.
|
||||
if not is_messageable_user(recipient):
|
||||
if recipient is None or not is_messageable_user(recipient):
|
||||
raise not_found("recipient")
|
||||
if recipient.id == user.id:
|
||||
raise refused(400, "share_own")
|
||||
exists = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == recipient.id
|
||||
)
|
||||
).first()
|
||||
if exists is None:
|
||||
db.add(DownloadShare(job_id=job_id, from_user_id=user.id, to_user_id=recipient.id))
|
||||
from app.notifications import create_notification
|
||||
|
||||
create_notification(
|
||||
db,
|
||||
user_id=recipient.id,
|
||||
type="download_shared",
|
||||
title=(job.display_name or "A download") + " was shared with you",
|
||||
# The client names the notice in the reader's language from `data`; the title is the
|
||||
# English fallback only.
|
||||
data={
|
||||
"kind": "download_shared",
|
||||
"from": user.email,
|
||||
"job_id": job_id,
|
||||
"title": job.display_name,
|
||||
},
|
||||
commit=False,
|
||||
)
|
||||
if service.share(db, job, user, recipient):
|
||||
db.commit()
|
||||
return {"ok": True, "shared_with": recipient.email}
|
||||
|
||||
@@ -673,22 +575,10 @@ def unshare_download(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
owned_or_404(
|
||||
db, DownloadJob, job_id, user, what="download"
|
||||
) # ownership guard (404s if not the caller's job)
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == email.strip().lower())
|
||||
).scalar_one_or_none()
|
||||
if recipient is not None:
|
||||
row = db.execute(
|
||||
select(DownloadShare).where(
|
||||
DownloadShare.job_id == job_id,
|
||||
DownloadShare.to_user_id == recipient.id,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
own_job(db, user, job_id) # ownership guard (404s if not the caller's job)
|
||||
recipient = user_by_email(db, email)
|
||||
if recipient is not None and service.unshare(db, job_id, recipient):
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@@ -701,54 +591,26 @@ def share_recipients(
|
||||
) -> list[dict]:
|
||||
"""Registered users this user can share a download with (excludes self + any non-messageable
|
||||
account — suspended or deactivated — so the picker never leaks those addresses)."""
|
||||
rows = (
|
||||
db.execute(
|
||||
select(User)
|
||||
.where(User.id != user.id, *messageable_clauses())
|
||||
.order_by(collated(User.email, lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
rows = messageable_others(db, user, lang_for_user(user))
|
||||
return [{"id": u.id, "email": u.email, "display_name": u.display_name} for u in rows]
|
||||
|
||||
|
||||
# --- public watch links (share by link) ----------------------------------------------------
|
||||
|
||||
|
||||
def _own_link(db: Session, user: User, link_id: int) -> DownloadLink:
|
||||
link = db.get(DownloadLink, link_id)
|
||||
def _own_link(db: Session, user: User, link_id: int) -> "DownloadLink":
|
||||
link = queries.own_link(db, user, link_id)
|
||||
if link is None:
|
||||
raise not_found("link")
|
||||
target = (
|
||||
db.get(DownloadJob, link.job_id)
|
||||
if link.job_id is not None
|
||||
else db.get(FileFolder, link.folder_id)
|
||||
)
|
||||
if target is None or target.user_id != user.id:
|
||||
raise not_found("link")
|
||||
return link
|
||||
|
||||
|
||||
def _link_expiry(days: int | None) -> datetime | None:
|
||||
# None/0/negative → no expiry; otherwise N days out, capped at ~10 years. The Pydantic model
|
||||
# already coerced/validated the type, so a non-number is a 422 before we get here.
|
||||
if not days or days <= 0:
|
||||
return None
|
||||
return datetime.now(timezone.utc) + timedelta(days=min(days, 3650))
|
||||
|
||||
|
||||
def _links_of(db: Session, *where) -> list[dict]:
|
||||
rows = db.execute(select(DownloadLink).where(*where).order_by(DownloadLink.id.desc()))
|
||||
return [linksmod.owner_view(row) for row in rows.scalars()]
|
||||
|
||||
|
||||
@router.get("/{job_id}/links")
|
||||
def list_links(
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
return _links_of(db, DownloadLink.job_id == job_id)
|
||||
own_job(db, user, job_id)
|
||||
return [linksmod.owner_view(link) for link in queries.links_to(db, job_id=job_id)]
|
||||
|
||||
|
||||
class CreateLinkIn(BaseModel):
|
||||
@@ -764,30 +626,18 @@ class CreateLinkIn(BaseModel):
|
||||
|
||||
def _new_link(
|
||||
db: Session, user: User, body: CreateLinkIn, **target: int
|
||||
) -> tuple[DownloadLink, str]:
|
||||
"""Make a link to `target` (`job_id=` or `folder_id=`, already checked to be the user's), and
|
||||
the token it is opened by — which is in hand only now."""
|
||||
pw = body.password.strip()
|
||||
token = linksmod.new_token()
|
||||
link = DownloadLink(
|
||||
**target,
|
||||
created_by=user.id,
|
||||
allow_download=body.allow_download,
|
||||
password_hash=hash_password(pw) if pw else None,
|
||||
expires_at=_link_expiry(body.expires_days),
|
||||
)
|
||||
linksmod.seal(link, token)
|
||||
db.add(link)
|
||||
db.flush()
|
||||
audit.record(
|
||||
) -> tuple["DownloadLink", str]:
|
||||
"""A link to `target` (`job_id=` or `folder_id=`, already checked to be the user's) and its
|
||||
token (`links.create`). The Pydantic model already coerced the expiry, so a non-number is a 422
|
||||
before this runs."""
|
||||
return linksmod.create(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_CREATE,
|
||||
target_type="download_link",
|
||||
target_id=link.id,
|
||||
after={**target, **_link_audit(link)},
|
||||
user,
|
||||
password=body.password.strip(),
|
||||
allow_download=body.allow_download,
|
||||
expires_days=body.expires_days,
|
||||
**target,
|
||||
)
|
||||
return link, token
|
||||
|
||||
|
||||
@router.post("/{job_id}/links")
|
||||
@@ -805,28 +655,20 @@ def create_link(
|
||||
return linksmod.owner_view(link, token=token)
|
||||
|
||||
|
||||
def _shareable_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""One of the user's files, held (FOR SHARE) for the rest of the transaction: a delete of it at
|
||||
the same moment (the user's, or the retention sweep's) waits and takes the new link with it,
|
||||
rather than failing the link's foreign key — or, done first, leaves a 404."""
|
||||
owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update(read=True)
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
def _shareable_job(db: Session, user: User, job_id: int) -> "DownloadJob":
|
||||
"""One of the user's files, held for the link's transaction (`service.hold_for_share`)."""
|
||||
job = service.hold_for_share(db, user, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder:
|
||||
def _shareable_folder(db: Session, user: User, folder_id: int) -> "FileFolder":
|
||||
"""One of the user's own folders. A source folder lets its files go on its own clock, and the Hub
|
||||
is the user's channel to a machine of theirs: neither is a shelf to hand out. Held (FOR SHARE)
|
||||
for the rest of the transaction, so a delete of it at the same moment waits and takes the new
|
||||
link with it, rather than failing the link's foreign key."""
|
||||
owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||
own_folder(db, user, folder_id)
|
||||
try:
|
||||
folder = folders.own_folder(db, user.id, folder_id)
|
||||
except folders.FolderError as e:
|
||||
@@ -840,8 +682,8 @@ def _shareable_folder(db: Session, user: User, folder_id: int) -> FileFolder:
|
||||
def list_folder_links(
|
||||
folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||
return _links_of(db, DownloadLink.folder_id == folder_id)
|
||||
own_folder(db, user, folder_id)
|
||||
return [linksmod.owner_view(link) for link in queries.links_to(db, folder_id=folder_id)]
|
||||
|
||||
|
||||
@router.post("/folders/{folder_id}/links")
|
||||
@@ -857,15 +699,6 @@ def create_folder_link(
|
||||
return linksmod.owner_view(link, token=token)
|
||||
|
||||
|
||||
def _link_audit(link: DownloadLink) -> dict:
|
||||
"""A link's settings as the audit records them: whether it has a password, never which."""
|
||||
return {
|
||||
"allow_download": link.allow_download,
|
||||
"has_password": link.password_hash is not None,
|
||||
"expires_at": iso(link.expires_at),
|
||||
}
|
||||
|
||||
|
||||
class UpdateLinkIn(BaseModel):
|
||||
allow_download: bool | None = None
|
||||
expires_days: int | None = None
|
||||
@@ -880,29 +713,15 @@ def update_link(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
link = _own_link(db, user, link_id)
|
||||
before = _link_audit(link)
|
||||
sent = body.model_fields_set
|
||||
if "allow_download" in sent:
|
||||
link.allow_download = bool(body.allow_download)
|
||||
if "expires_days" in sent:
|
||||
link.expires_at = _link_expiry(body.expires_days)
|
||||
if "password" in sent:
|
||||
pw = (body.password or "").strip()
|
||||
link.password_hash = hash_password(pw) if pw else None
|
||||
# Bump the version so any grant issued under the previous password stops working now, rather
|
||||
# than living out its 6h TTL (setting/clearing/changing the password all count as a rotation).
|
||||
link.password_version += 1
|
||||
after = _link_audit(link)
|
||||
if after != before or "password" in sent:
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_UPDATE,
|
||||
target_type="download_link",
|
||||
target_id=link_id,
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
linksmod.update(
|
||||
db,
|
||||
user,
|
||||
link,
|
||||
allow_download=body.allow_download,
|
||||
expires_days=body.expires_days,
|
||||
password=body.password,
|
||||
sent=frozenset(body.model_fields_set),
|
||||
)
|
||||
db.commit()
|
||||
return linksmod.owner_view(link)
|
||||
|
||||
@@ -911,16 +730,7 @@ def update_link(
|
||||
def revoke_link(
|
||||
link_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
link = _own_link(db, user, link_id)
|
||||
db.delete(link)
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_LINK_REVOKE,
|
||||
target_type="download_link",
|
||||
target_id=link_id,
|
||||
after={"job_id": link.job_id} if link.job_id is not None else {"folder_id": link.folder_id},
|
||||
)
|
||||
linksmod.revoke(db, user, _own_link(db, user, link_id))
|
||||
db.commit()
|
||||
return {"ok": True, "deleted": link_id}
|
||||
|
||||
@@ -932,20 +742,9 @@ def revoke_link(
|
||||
def admin_list_downloads(
|
||||
admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
jobs = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(MAX_LIST)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assets = assets_for(db, jobs)
|
||||
emails = {
|
||||
u.id: u.email
|
||||
for u in db.execute(select(User).where(User.id.in_({j.user_id for j in jobs}))).scalars()
|
||||
}
|
||||
jobs = queries.recent_jobs(db)
|
||||
assets = queries.assets_for(db, jobs)
|
||||
emails = emails_of(db, {j.user_id for j in jobs})
|
||||
out = []
|
||||
for j in jobs:
|
||||
data = serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None)
|
||||
@@ -957,27 +756,11 @@ def admin_list_downloads(
|
||||
|
||||
@admin_router.get("/storage")
|
||||
def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
# Every file on disk (`quota.ON_DISK`): ready ones, and uploads waiting for or failed in their
|
||||
# conversion — the same disk the footprint and the GC cap count.
|
||||
on_disk = db.execute(
|
||||
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).one()
|
||||
total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar()
|
||||
files_on_disk, total_bytes, total_assets = queries.storage_totals(db)
|
||||
# EVERY user, not only those holding a file: a quota is set before anyone downloads.
|
||||
emails: dict[int, str] = {
|
||||
uid: email
|
||||
for uid, email in db.execute(select(User.id, User.email).order_by(User.email)).tuples()
|
||||
}
|
||||
emails = emails_of(db)
|
||||
uids = list(emails)
|
||||
# What each one's Files lists: their finished files.
|
||||
files: dict[int, int] = {
|
||||
uid: n
|
||||
for uid, n in db.execute(
|
||||
select(DownloadJob.user_id, func.count())
|
||||
.where(DownloadJob.status == JobStatus.DONE)
|
||||
.group_by(DownloadJob.user_id)
|
||||
).tuples()
|
||||
}
|
||||
files = queries.done_files_per_user(db)
|
||||
|
||||
def row(uid: int) -> dict:
|
||||
lim = quota.resolve(db, uid)
|
||||
@@ -991,8 +774,8 @@ def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_d
|
||||
}
|
||||
|
||||
return {
|
||||
"files_on_disk": on_disk[0],
|
||||
"total_bytes": int(on_disk[1]),
|
||||
"files_on_disk": files_on_disk,
|
||||
"total_bytes": total_bytes,
|
||||
"total_assets": total_assets,
|
||||
"total_cap_bytes": sysconfig.get_int(db, "download_total_max_bytes"),
|
||||
"per_user": [row(uid) for uid in uids],
|
||||
@@ -1004,10 +787,9 @@ def admin_get_quota(
|
||||
user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
lim = quota.resolve(db, user_id)
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
return {
|
||||
"user_id": user_id,
|
||||
"custom": row is not None,
|
||||
"custom": queries.custom_quota(db, user_id) is not None,
|
||||
"max_bytes": lim.max_bytes,
|
||||
"max_concurrent": lim.max_concurrent,
|
||||
"max_jobs": lim.max_jobs,
|
||||
@@ -1032,25 +814,14 @@ def admin_set_quota(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
get_or_404(db, User, user_id, what="user")
|
||||
cur = quota.resolve(db, user_id)
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is None:
|
||||
row = DownloadQuota(
|
||||
user_id=user_id,
|
||||
max_bytes=cur.max_bytes,
|
||||
max_concurrent=cur.max_concurrent,
|
||||
max_jobs=cur.max_jobs,
|
||||
unlimited=cur.unlimited,
|
||||
)
|
||||
db.add(row)
|
||||
if body.max_bytes is not None:
|
||||
row.max_bytes = max(0, body.max_bytes)
|
||||
if body.max_concurrent is not None:
|
||||
row.max_concurrent = max(1, body.max_concurrent)
|
||||
if body.max_jobs is not None:
|
||||
row.max_jobs = max(1, body.max_jobs)
|
||||
if body.unlimited is not None:
|
||||
row.unlimited = body.unlimited
|
||||
quota.set_custom(
|
||||
db,
|
||||
user_id,
|
||||
max_bytes=body.max_bytes,
|
||||
max_concurrent=body.max_concurrent,
|
||||
max_jobs=body.max_jobs,
|
||||
unlimited=body.unlimited,
|
||||
)
|
||||
db.commit()
|
||||
return admin_get_quota(user_id, admin, db)
|
||||
|
||||
@@ -1059,8 +830,6 @@ def admin_set_quota(
|
||||
def admin_reset_quota(
|
||||
user_id: int, admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
if quota.reset_custom(db, user_id):
|
||||
db.commit()
|
||||
return admin_get_quota(user_id, admin, db)
|
||||
|
||||
+51
-183
@@ -1,9 +1,7 @@
|
||||
from datetime import date, datetime, timezone
|
||||
from datetime import date
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import StaleDataError
|
||||
|
||||
@@ -11,34 +9,10 @@ from app import quota
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.mailtext import lang_for_user
|
||||
from app.feedquery import (
|
||||
decode_cursor,
|
||||
encode_cursor,
|
||||
feed_summary,
|
||||
feed_years,
|
||||
filtered_query,
|
||||
keyset_predicate,
|
||||
order_by,
|
||||
rank_window_start,
|
||||
serialize,
|
||||
sort_keys,
|
||||
windowed_rank,
|
||||
windowed_relevance_rows,
|
||||
)
|
||||
from app.models import (
|
||||
ChannelTag,
|
||||
Tag,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.routes._common import (
|
||||
ProgressIn,
|
||||
get_or_404,
|
||||
kept_position,
|
||||
not_found,
|
||||
youtube_refusal,
|
||||
)
|
||||
from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years
|
||||
from app.models import User
|
||||
from app.reader import queries, service
|
||||
from app.routes._common import ProgressIn, kept_position, not_found, youtube_refusal
|
||||
from app.sync.videos import parse_iso8601_duration
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
@@ -108,49 +82,20 @@ def get_feed(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
lang = lang_for_user(user)
|
||||
keys = sort_keys(sort, seed, rank_expr, lang)
|
||||
|
||||
vals, cursor_id, window = None, None, None
|
||||
if cursor:
|
||||
# The read-model raises a plain ValueError on a malformed cursor; translate it to the
|
||||
# HTTP 400 here so feedquery stays web-framework-free.
|
||||
try:
|
||||
vals, cursor_id, window = decode_cursor(keys, cursor)
|
||||
except ValueError:
|
||||
# `from None`: the ValueError is an expected bad-input signal, not an error to chain.
|
||||
raise refused(400, "invalid_cursor") from None
|
||||
ranked = sort == "relevance" and rank_expr is not None
|
||||
if ranked and not cursor:
|
||||
window = rank_window_start(db, query)
|
||||
if ranked and window is not None:
|
||||
keys = sort_keys(sort, seed, windowed_rank(rank_expr, window), lang)
|
||||
|
||||
# Expose each key column on the row so we can build the next cursor from the last item.
|
||||
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
||||
if vals is not None:
|
||||
for k, v in zip(keys, vals, strict=True):
|
||||
k["value"] = v
|
||||
if ranked and window is not None:
|
||||
rows = windowed_relevance_rows(db, query, keys, window, cursor_id, limit + 1)
|
||||
else:
|
||||
query = query.order_by(*order_by(keys))
|
||||
if cursor_id is not None:
|
||||
query = query.where(keyset_predicate(keys, cursor_id))
|
||||
rows = db.execute(query.limit(limit + 1)).all()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
return {
|
||||
"items": [serialize(r) for r in page],
|
||||
"next_cursor": (
|
||||
encode_cursor(keys, page[-1], window if ranked else None)
|
||||
if (has_more and page)
|
||||
else None
|
||||
),
|
||||
"has_more": has_more,
|
||||
"limit": limit,
|
||||
}
|
||||
try:
|
||||
return feed_page(
|
||||
db,
|
||||
user,
|
||||
params,
|
||||
sort=sort,
|
||||
seed=seed,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
lang=lang_for_user(user),
|
||||
)
|
||||
except BadCursor:
|
||||
# `from None`: a malformed cursor is an expected bad-input signal, not an error to chain.
|
||||
raise refused(400, "invalid_cursor") from None
|
||||
|
||||
|
||||
@router.get("/feed/count")
|
||||
@@ -162,9 +107,7 @@ def get_feed_count(
|
||||
# SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference
|
||||
# oracle that test_feed_summary_matches_count_plus_facets asserts /feed/summary's `count` against;
|
||||
# remove both together with that test if this endpoint is ever retired.
|
||||
query, _rank = filtered_query(db, user, **params)
|
||||
total = db.scalar(select(func.count()).select_from(query.subquery()))
|
||||
return {"count": total or 0}
|
||||
return {"count": feed_count(db, user, **params)}
|
||||
|
||||
|
||||
@router.get("/facets")
|
||||
@@ -186,31 +129,7 @@ def get_facets(
|
||||
SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference
|
||||
oracle for test_feed_summary_matches_count_plus_facets (it asserts /feed/summary's `facets`
|
||||
against these `counts`); retire the two together."""
|
||||
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||
counts: dict[int, int] = {}
|
||||
for category in ("language", "topic", "other"):
|
||||
# Disjunctive (OR) facets drop the category's own selections so its chips keep
|
||||
# independent counts (you can OR more of them in). Conjunctive (AND, topics only)
|
||||
# keeps them applied, so each remaining chip narrows to channels that ALSO have all
|
||||
# already-selected topics — and tags that can't co-occur drop to zero (hidden).
|
||||
conjunctive = params.get("tag_modes", {}).get(category) == "and"
|
||||
base, _rank = filtered_query(
|
||||
db,
|
||||
user,
|
||||
**{**params, "exclude_tag_category": None if conjunctive else category},
|
||||
)
|
||||
channels = base.with_only_columns(Video.channel_id).distinct().subquery()
|
||||
rows = db.execute(
|
||||
select(ChannelTag.tag_id, func.count(func.distinct(ChannelTag.channel_id)))
|
||||
.select_from(channels)
|
||||
.join(ChannelTag, ChannelTag.channel_id == channels.c.channel_id)
|
||||
.join(Tag, and_(Tag.id == ChannelTag.tag_id, Tag.category == category))
|
||||
.where(visible)
|
||||
.group_by(ChannelTag.tag_id)
|
||||
).all()
|
||||
for tag_id, count in rows:
|
||||
counts[tag_id] = count
|
||||
return {"counts": counts}
|
||||
return {"counts": facet_counts(db, user, **params)}
|
||||
|
||||
|
||||
@router.get("/feed/summary")
|
||||
@@ -239,6 +158,11 @@ def get_feed_years(
|
||||
return {"years": feed_years(db, user, **params)}
|
||||
|
||||
|
||||
def _video_or_404(db: Session, video_id: str) -> None:
|
||||
if queries.video(db, video_id) is None:
|
||||
raise not_found("video")
|
||||
|
||||
|
||||
class VideoStateIn(BaseModel):
|
||||
status: str
|
||||
|
||||
@@ -253,50 +177,17 @@ def set_video_state(
|
||||
status = body.status
|
||||
if status not in VALID_STATES:
|
||||
raise refused(400, "invalid_status")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
_video_or_404(db, video_id)
|
||||
if status == "new":
|
||||
# Un-marking: keep the row if it still holds a resume position (un-marking
|
||||
# "watched" should restore the in-progress state, not wipe where the user left
|
||||
# off); otherwise drop it. The in-app player fires this alongside a progress
|
||||
# checkpoint that may DELETE the same row, so tolerate it vanishing under us.
|
||||
row = db.execute(
|
||||
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
|
||||
).scalar_one_or_none()
|
||||
if row is not None:
|
||||
if row.position_seconds:
|
||||
row.status = "new"
|
||||
row.watched_at = None
|
||||
else:
|
||||
db.delete(row)
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: the documented lost race — a concurrent player checkpoint already removed
|
||||
# or moved this row. The state the caller asked for is the state that holds either way.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
try:
|
||||
service.unmark(db, user, video_id)
|
||||
db.commit()
|
||||
# silent-ok: the documented lost race — a concurrent player checkpoint already removed or
|
||||
# moved this row (`service.unmark`). The state the caller asked for holds either way.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"ok": True, "video_id": video_id, "status": "new"}
|
||||
|
||||
# watched / hidden: atomic upsert. The player marks "watched" at end-of-playback at
|
||||
# the same instant a progress checkpoint may DELETE the in-progress row; a plain
|
||||
# SELECT-then-UPDATE then raises StaleDataError ("0 rows matched") on that race. An
|
||||
# INSERT … ON CONFLICT DO UPDATE keyed on uq_user_video is immune to it.
|
||||
set_: dict = {"status": status, "updated_at": now}
|
||||
if status == "watched":
|
||||
set_["watched_at"] = now # hidden keeps any existing watched_at
|
||||
stmt = (
|
||||
pg_insert(VideoState)
|
||||
.values(
|
||||
user_id=user.id,
|
||||
video_id=video_id,
|
||||
status=status,
|
||||
watched_at=now if status == "watched" else None,
|
||||
)
|
||||
.on_conflict_do_update(constraint="uq_user_video", set_=set_)
|
||||
)
|
||||
db.execute(stmt)
|
||||
service.mark(db, user, video_id, status)
|
||||
db.commit()
|
||||
return {"ok": True, "video_id": video_id, "status": status}
|
||||
|
||||
@@ -310,16 +201,12 @@ def clear_video_state(
|
||||
"""Reset a video to pristine for this user — drop the whole VideoState row (status,
|
||||
watch position and watched_at), as if it had never been opened. Saved/playlist membership
|
||||
lives elsewhere and is untouched."""
|
||||
row = db.execute(
|
||||
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
|
||||
).scalar_one_or_none()
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
try:
|
||||
service.clear_state(db, user, video_id)
|
||||
db.commit()
|
||||
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
|
||||
|
||||
@@ -335,36 +222,17 @@ def set_video_progress(
|
||||
partially-watched video can render a progress bar and match the "in progress" filter.
|
||||
Trivially-early and near-finished positions clear the position rather than store it."""
|
||||
position = kept_position(body)
|
||||
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
row = db.execute(
|
||||
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
|
||||
).scalar_one_or_none()
|
||||
|
||||
_video_or_404(db, video_id)
|
||||
if not position:
|
||||
# Nothing meaningful to store: drop a status-less row entirely, otherwise just
|
||||
# zero the position (a saved/hidden video keeps its status).
|
||||
if row is not None:
|
||||
if row.status == "new":
|
||||
db.delete(row)
|
||||
else:
|
||||
row.position_seconds = 0
|
||||
row.progress_updated_at = datetime.now(timezone.utc)
|
||||
# A concurrent state change (e.g. the end-of-playback "watched" mark) may
|
||||
# have removed/updated this row already — tolerate the lost race.
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: exactly the race the comment above describes; the response stays true.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
try:
|
||||
service.set_progress(db, user, video_id, 0)
|
||||
db.commit()
|
||||
# silent-ok: a concurrent state change (the end-of-playback "watched" mark) removed or
|
||||
# updated this row already — the lost race; the response stays true.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": 0}
|
||||
|
||||
if row is None:
|
||||
row = VideoState(user_id=user.id, video_id=video_id)
|
||||
db.add(row)
|
||||
row.position_seconds = position
|
||||
row.progress_updated_at = datetime.now(timezone.utc)
|
||||
service.set_progress(db, user, video_id, position)
|
||||
db.commit()
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": position}
|
||||
|
||||
@@ -382,7 +250,7 @@ def get_video_detail(
|
||||
our DB (e.g. a YouTube link inside another video's description that the in-app
|
||||
player navigated to) is resolved via the YouTube API (videos.list, 1 unit,
|
||||
attributed to the requesting user)."""
|
||||
v = db.get(Video, video_id)
|
||||
v = queries.video(db, video_id)
|
||||
if v is not None:
|
||||
return {
|
||||
"id": v.id,
|
||||
|
||||
+91
-244
@@ -5,8 +5,10 @@ the per-user layer (name, folder, kept, expiry, shares and links all hang off it
|
||||
bytes. Folders are rows (`downloads.folders`), so a rename or a move never touches the disk.
|
||||
|
||||
In-flight jobs are not here: they are the download tray's (`routes.downloads`). The helpers above
|
||||
the routes are shared with that module — serialization, the ownership locks, the upload spool —
|
||||
because a download and a library file are one row at two moments.
|
||||
the routes are shared with that module — serialization, the ownership checks, the upload spool —
|
||||
because a download and a library file are one row at two moments. What is read comes from
|
||||
`downloads.queries`, what is written goes through Capture's service (`service`, `folders`); this
|
||||
module validates, answers in HTTP and commits.
|
||||
|
||||
A machine with an API token (`app.apitokens`) may use the routes that take `files_caller`: list the
|
||||
folders and files, fetch a file and its hash, upload, delete — each only in a folder its token was
|
||||
@@ -19,51 +21,37 @@ import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO, Literal
|
||||
from typing import TYPE_CHECKING, BinaryIO, Literal
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, sysconfig
|
||||
from app.apitokens import FilesCaller, Right, files_caller
|
||||
from app.audit import AuditAction
|
||||
from app.auth import current_user
|
||||
from app.collation import collated
|
||||
from app.config import settings
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.downloads import folders, preview, quota, service, storage
|
||||
from app.downloads import folders, preview, queries, quota, service, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads.status import JobStatus
|
||||
from app.mailtext import lang_for_user
|
||||
from app.models import (
|
||||
DISPLAY_TEXT_MAX,
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
DownloadShare,
|
||||
FileFolder,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
User,
|
||||
display_text,
|
||||
)
|
||||
from app.models import DISPLAY_TEXT_MAX, User, display_text
|
||||
from app.refusal import refused
|
||||
from app.routes._common import capped_stream_to, not_found, owned_or_404
|
||||
from app.utils import iso, like_escape
|
||||
from app.routes._common import capped_stream_to, not_found
|
||||
from app.utils import iso
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||
|
||||
router = APIRouter(prefix="/api/files", tags=["files"])
|
||||
|
||||
# Cap the listing so a heavy library can't return an unbounded result set (the old Download Center's
|
||||
# own cap; far beyond any real library).
|
||||
MAX_LIST = 500
|
||||
# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
|
||||
_MAX_RETENTION_DAYS = 3650
|
||||
_HASH_CHUNK = 1 << 20
|
||||
@@ -75,7 +63,7 @@ _HASHING = threading.BoundedSemaphore(_HASHERS)
|
||||
# --- shared with routes.downloads -------------------------------------------------------
|
||||
|
||||
|
||||
def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
def serialize(job: "DownloadJob", asset: "MediaAsset | None") -> dict:
|
||||
ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
|
||||
return {
|
||||
"id": job.id,
|
||||
@@ -135,58 +123,18 @@ def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
|
||||
ids = {j.asset_id for j in jobs if j.asset_id}
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
|
||||
return {a.id: a for a in rows}
|
||||
|
||||
|
||||
def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
|
||||
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
|
||||
ids = [j.id for j in jobs]
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(
|
||||
select(LibraryState.job_id, LibraryState.position_seconds).where(
|
||||
LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
|
||||
)
|
||||
).all()
|
||||
return {job_id: pos for job_id, pos in rows}
|
||||
|
||||
|
||||
def serialize_job(db: Session, job: DownloadJob) -> dict:
|
||||
def serialize_job(db: Session, job: "DownloadJob") -> dict:
|
||||
"""Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
|
||||
every single-job handler shares."""
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
return serialize(job, asset)
|
||||
return serialize(job, queries.asset_of(db, job))
|
||||
|
||||
|
||||
def own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
||||
or writes the status those decide on (pause). The job is re-read under a row lock, so two
|
||||
overlapping requests for one job run one after the other and the second decides on the first's
|
||||
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||
just released back into a holder; either can free a file another job still holds.
|
||||
|
||||
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||
(`service.finalize_if_running`), so the two can never deadlock."""
|
||||
while True:
|
||||
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
if job.asset_id == asset_id:
|
||||
return job
|
||||
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||
def own_job_for_update(db: Session, user: User, job_id: int) -> "DownloadJob":
|
||||
"""One of the user's jobs under its row lock (`service.lock_own_job`), or a 404."""
|
||||
job = service.lock_own_job(db, user, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
def quota_detail(e: quota.QuotaExceeded) -> dict:
|
||||
@@ -322,30 +270,32 @@ async def spool_upload(
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""A file the caller may read: their own, or one shared with them."""
|
||||
job = db.get(DownloadJob, job_id)
|
||||
def accessible_job(db: Session, user: User, job_id: int) -> "DownloadJob":
|
||||
"""A file the caller may read (`queries.accessible_job`: their own, or shared with them)."""
|
||||
job = queries.accessible_job(db, user, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
if job.user_id == user.id:
|
||||
return job
|
||||
shared = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
|
||||
)
|
||||
).first()
|
||||
if shared is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob:
|
||||
"""A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's
|
||||
own in a folder it may read."""
|
||||
if caller.grants is None:
|
||||
return accessible_job(db, caller.user, job_id)
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None or job.user_id != caller.user.id or not caller.may(job.folder_id, Right.READ):
|
||||
def own_job(db: Session, user: User, job_id: int) -> "DownloadJob":
|
||||
job = queries.own_job(db, user, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
def own_folder(db: Session, user: User, folder_id: int) -> "FileFolder":
|
||||
folder = queries.own_folder(db, user, folder_id)
|
||||
if folder is None:
|
||||
raise not_found("folder")
|
||||
return folder
|
||||
|
||||
|
||||
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> "DownloadJob":
|
||||
"""A file `caller` may read (`queries.readable_job`), or a 404."""
|
||||
job = queries.readable_job(db, caller, job_id)
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
@@ -353,7 +303,7 @@ def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob:
|
||||
# --- folders -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _serialize_folder(folder: FileFolder, count: int, size: int, linked: bool = False) -> dict:
|
||||
def _serialize_folder(folder: "FileFolder", count: int, size: int, linked: bool = False) -> dict:
|
||||
return {
|
||||
"id": folder.id,
|
||||
"kind": folder.kind,
|
||||
@@ -369,18 +319,6 @@ def _serialize_folder(folder: FileFolder, count: int, size: int, linked: bool =
|
||||
}
|
||||
|
||||
|
||||
def _linked(db: Session, folder_ids: list[int]) -> set[int]:
|
||||
"""Which of these folders a live public link opens (one not past its expiry)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = db.execute(
|
||||
select(DownloadLink.folder_id).where(
|
||||
DownloadLink.folder_id.in_(folder_ids),
|
||||
or_(DownloadLink.expires_at.is_(None), DownloadLink.expires_at > now),
|
||||
)
|
||||
).scalars()
|
||||
return {folder_id for folder_id in rows if folder_id is not None}
|
||||
|
||||
|
||||
@router.get("/folders")
|
||||
def list_folders(
|
||||
caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
||||
@@ -388,23 +326,9 @@ def list_folders(
|
||||
"""The caller's folders with how many files each holds and their size (a token: the ones it may
|
||||
list). A source folder its files have all left stays (count 0): the client decides whether to
|
||||
show it."""
|
||||
done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
|
||||
stmt = select(
|
||||
FileFolder,
|
||||
func.count(DownloadJob.id),
|
||||
func.coalesce(func.sum(MediaAsset.size_bytes), 0),
|
||||
).where(FileFolder.user_id == caller.user.id)
|
||||
listable = caller.folders_with(Right.LIST)
|
||||
if listable is not None:
|
||||
stmt = stmt.where(FileFolder.id.in_(listable))
|
||||
rows = db.execute(
|
||||
stmt.outerjoin(DownloadJob, done)
|
||||
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.group_by(FileFolder.id)
|
||||
.order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
|
||||
).all()
|
||||
linked = _linked(db, [f.id for f, _, _ in rows])
|
||||
return [_serialize_folder(f, count, int(size), f.id in linked) for f, count, size in rows]
|
||||
rows = queries.folders_with_totals(db, caller.user, caller.folders_with(Right.LIST))
|
||||
linked = queries.linked_folders(db, [f.id for f, _, _ in rows])
|
||||
return [_serialize_folder(f, count, size, f.id in linked) for f, count, size in rows]
|
||||
|
||||
|
||||
class FolderIn(BaseModel):
|
||||
@@ -422,13 +346,10 @@ def _folder_name(raw: str) -> str:
|
||||
def create_folder(
|
||||
body: FolderIn, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
folder = FileFolder(user_id=user.id, kind=folders.FolderKind.OWN, name=_folder_name(body.name))
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(folder)
|
||||
db.flush()
|
||||
except IntegrityError as e: # the unique (user, lower(name)): one folder per name
|
||||
raise refused(409, "file_folder_name_taken") from e
|
||||
folder = folders.create_own(db, user.id, _folder_name(body.name))
|
||||
except folders.FolderError as e:
|
||||
raise refused(409, e.reason) from e
|
||||
audit.record(
|
||||
db, user.id, AuditAction.FILE_FOLDER_CREATE, target_type="file_folder", target_id=folder.id
|
||||
)
|
||||
@@ -451,17 +372,13 @@ def update_folder(
|
||||
) -> dict:
|
||||
"""Rename one of the caller's own folders (a source folder's name is its origin's), and/or set
|
||||
how long any of their folders keeps a file (`folders.set_retention`)."""
|
||||
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||
folder = own_folder(db, user, folder_id)
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
if folder.kind != folders.FolderKind.OWN:
|
||||
raise refused(409, "file_folder_not_own")
|
||||
folder.name = _folder_name(body.name or "")
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.flush()
|
||||
except IntegrityError as e:
|
||||
raise refused(409, "file_folder_name_taken") from e
|
||||
folders.rename(db, folder, _folder_name(body.name or ""))
|
||||
except folders.FolderError as e:
|
||||
raise refused(409, e.reason) from e
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
@@ -486,18 +403,9 @@ def update_folder(
|
||||
after={"retention_days": body.retention_days},
|
||||
)
|
||||
db.commit()
|
||||
count, size = _folder_totals(db, folder.id)
|
||||
return _serialize_folder(folder, count, size, folder.id in _linked(db, [folder.id]))
|
||||
|
||||
|
||||
def _folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
|
||||
count, size = db.execute(
|
||||
select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
|
||||
.select_from(DownloadJob)
|
||||
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
|
||||
).one()
|
||||
return count, int(size)
|
||||
count, size = queries.folder_totals(db, folder.id)
|
||||
linked = folder.id in queries.linked_folders(db, [folder.id])
|
||||
return _serialize_folder(folder, count, size, linked)
|
||||
|
||||
|
||||
@router.delete("/folders/{folder_id}")
|
||||
@@ -505,7 +413,7 @@ def delete_folder(
|
||||
folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Delete one of the caller's own folders; its files go back to their source folders."""
|
||||
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||
folder = own_folder(db, user, folder_id)
|
||||
try:
|
||||
moved = folders.delete_own(db, folder)
|
||||
except folders.FolderError as e:
|
||||
@@ -547,13 +455,7 @@ def remove_shared_with_me(
|
||||
"""Remove a file that was shared WITH this user from their 'Shared with me' list. Deletes
|
||||
only the recipient's share grant — the owner's job and the physical file are untouched (this is
|
||||
a per-user dismissal, not a delete)."""
|
||||
row = db.execute(
|
||||
select(DownloadShare).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
if service.unshare(db, job_id, user):
|
||||
db.commit()
|
||||
return {"ok": True, "removed": job_id}
|
||||
|
||||
@@ -582,46 +484,22 @@ def list_files(
|
||||
raise refused(403, "api_token_scope")
|
||||
if folder_id is not None and folder_id not in listable:
|
||||
raise not_found("folder")
|
||||
stmt = (
|
||||
select(DownloadJob)
|
||||
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.where(DownloadJob.status == JobStatus.DONE)
|
||||
)
|
||||
shared = scope == "shared"
|
||||
if shared:
|
||||
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
|
||||
DownloadShare.to_user_id == user.id
|
||||
)
|
||||
else:
|
||||
stmt = stmt.where(DownloadJob.user_id == user.id)
|
||||
if folder_id is not None:
|
||||
stmt = stmt.where(DownloadJob.folder_id == folder_id)
|
||||
elif listable is not None:
|
||||
stmt = stmt.where(DownloadJob.folder_id.in_(listable))
|
||||
name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
|
||||
term = q.strip()
|
||||
if term:
|
||||
like = f"%{like_escape(term)}%"
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
name.ilike(like),
|
||||
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
|
||||
)
|
||||
)
|
||||
# A file is added to its owner's list when it is done, not when it was asked for (a download that
|
||||
# failed for days and then went through is new); one shared with the caller, when it was shared.
|
||||
key = {
|
||||
"added": DownloadShare.created_at if shared else DownloadJob.ready_at,
|
||||
"name": collated(name, lang_for_user(user)),
|
||||
"size": MediaAsset.size_bytes,
|
||||
}[sort]
|
||||
first = key.asc() if order == "asc" else key.desc()
|
||||
tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
|
||||
jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all()
|
||||
assets = assets_for(db, jobs)
|
||||
positions = positions_for(db, user, jobs)
|
||||
shared_at = _shared_at(db, user, jobs) if shared else {}
|
||||
sources = source_keys(db, jobs)
|
||||
jobs = queries.library(
|
||||
db,
|
||||
user,
|
||||
folder_id=folder_id,
|
||||
listable=listable,
|
||||
shared=shared,
|
||||
term=q.strip(),
|
||||
sort=sort,
|
||||
order=order,
|
||||
lang=lang_for_user(user),
|
||||
)
|
||||
assets = queries.assets_for(db, jobs)
|
||||
positions = queries.positions_for(db, user, jobs)
|
||||
shared_at = queries.shared_at(db, user, jobs) if shared else {}
|
||||
sources = queries.source_keys(db, jobs)
|
||||
return [
|
||||
{
|
||||
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
|
||||
@@ -634,36 +512,6 @@ def list_files(
|
||||
]
|
||||
|
||||
|
||||
def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]:
|
||||
"""Where each file came from (its source folder's key), for the list's Source column, the
|
||||
"back to its source folder" move and the download tray's site: a file in a source folder has its folder's key; one moved into
|
||||
an own folder (or someone else's, shared) is traced back by `folders.source_key_for`."""
|
||||
ids = {j.folder_id for j in jobs if j.folder_id is not None}
|
||||
by_id = (
|
||||
{f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
|
||||
if ids
|
||||
else {}
|
||||
)
|
||||
out: dict[int, str] = {}
|
||||
for j in jobs:
|
||||
folder = by_id.get(j.folder_id) if j.folder_id is not None else None
|
||||
if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key:
|
||||
out[j.id] = folder.source_key
|
||||
else:
|
||||
out[j.id] = folders.source_key_for(db, j)
|
||||
return out
|
||||
|
||||
|
||||
def _shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]:
|
||||
rows = db.execute(
|
||||
select(DownloadShare.job_id, DownloadShare.created_at).where(
|
||||
DownloadShare.to_user_id == user.id,
|
||||
DownloadShare.job_id.in_([j.id for j in jobs]),
|
||||
)
|
||||
).all()
|
||||
return {job_id: at for job_id, at in rows}
|
||||
|
||||
|
||||
_MAX_EXTRA_LINKS = 8
|
||||
|
||||
|
||||
@@ -711,18 +559,20 @@ def update_file(
|
||||
"""Rename a file and its channel, set its reference links (all display-only: the file on disk
|
||||
keeps its own name), move it to another folder (`folders.move`), or keep it until deleted /
|
||||
hand it back to its folder's retention."""
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
job = own_job(db, user, job_id)
|
||||
sent = body.model_fields_set
|
||||
display: dict[str, object] = {}
|
||||
if "name" in sent:
|
||||
job.display_name = display_text(body.name)
|
||||
display["display_name"] = display_text(body.name)
|
||||
if "uploader" in sent:
|
||||
job.display_uploader = display_text(body.uploader)
|
||||
display["display_uploader"] = display_text(body.uploader)
|
||||
if "uploader_url" in sent:
|
||||
job.display_uploader_url = _clean_url(body.uploader_url)
|
||||
display["display_uploader_url"] = _clean_url(body.uploader_url)
|
||||
if "extra_links" in sent:
|
||||
raw = body.extra_links or []
|
||||
cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS]
|
||||
job.extra_links = cleaned or None
|
||||
display["extra_links"] = cleaned or None
|
||||
service.set_display(job, **display)
|
||||
if "folder_id" in sent:
|
||||
if body.folder_id is None:
|
||||
raise refused(400, "file_folder_gone")
|
||||
@@ -768,7 +618,8 @@ def delete_file(
|
||||
target_id=job_id,
|
||||
summary=caller.via,
|
||||
)
|
||||
service.delete(db, job) # commits, the audit row with it
|
||||
service.delete(db, job)
|
||||
db.commit() # the audit row with it
|
||||
return {"ok": True, "deleted": job_id}
|
||||
|
||||
|
||||
@@ -787,7 +638,7 @@ def upload_file(
|
||||
return serialize_job(db, ingest_spooled(db, upload))
|
||||
|
||||
|
||||
def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
|
||||
def ingest_spooled(db: Session, upload: SpooledUpload) -> "DownloadJob":
|
||||
"""The spooled upload into the library (`upload.ingest_upload`, which commits its own rows),
|
||||
its refusals answered as HTTP."""
|
||||
if upload.size == 0:
|
||||
@@ -811,8 +662,8 @@ def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
|
||||
return job
|
||||
|
||||
|
||||
def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
def _ready_file(db: Session, job: "DownloadJob") -> tuple["MediaAsset", Path]:
|
||||
asset = queries.asset_of(db, job)
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise refused(409, "download_not_ready")
|
||||
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||
@@ -821,7 +672,7 @@ def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
|
||||
return asset, path
|
||||
|
||||
|
||||
def serve_file(db: Session, job: DownloadJob) -> FileResponse:
|
||||
def serve_file(db: Session, job: "DownloadJob") -> FileResponse:
|
||||
"""The file of `job` (the caller may read it), range-aware, named by the user's display name.
|
||||
|
||||
The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
|
||||
@@ -839,7 +690,7 @@ def serve_file(db: Session, job: DownloadJob) -> FileResponse:
|
||||
if asset.sha256:
|
||||
digest = base64.b64encode(bytes.fromhex(asset.sha256)).decode()
|
||||
headers["Repr-Digest"] = f"sha-256=:{digest}:"
|
||||
asset.last_access_at = datetime.now(timezone.utc) # the handler commits it
|
||||
service.touch(asset) # the handler commits it
|
||||
media_type = storage.served_media_type(asset.media_kind, asset.container)
|
||||
if media_type is None:
|
||||
return FileResponse(
|
||||
@@ -894,13 +745,13 @@ def preview_file(
|
||||
text = preview.text_utf8(head)
|
||||
if text is None:
|
||||
raise refused(415, "file_no_preview")
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
service.touch(asset)
|
||||
db.commit()
|
||||
return Response(text, media_type=preview.media_type(kind, asset.container), headers=headers)
|
||||
filename = storage.download_filename(
|
||||
job.display_name or asset.title or asset.source_ref, asset.container, path
|
||||
)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
service.touch(asset)
|
||||
db.commit()
|
||||
return FileResponse(
|
||||
path,
|
||||
@@ -936,10 +787,6 @@ def get_sha256(
|
||||
finally:
|
||||
_HASHING.release()
|
||||
digest = h.hexdigest()
|
||||
db.execute(
|
||||
update(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
|
||||
.values(sha256=digest)
|
||||
)
|
||||
service.store_sha256(db, asset_id, digest)
|
||||
db.commit()
|
||||
return {"sha256": digest}
|
||||
|
||||
+61
-304
@@ -3,31 +3,34 @@ CRUD (create / rename / delete, add / remove / reorder items) plus YouTube two-w
|
||||
mirror the user's own YouTube playlists in (`/sync-youtube`) and push/revert local edits back
|
||||
out (`/{id}/push-plan`, `/{id}/revert-youtube`), behind the optional write scope."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, select, and_
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import quota
|
||||
from app.auth import current_user, has_read_scope, has_write_scope, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
|
||||
from app.feedquery import serialize
|
||||
from app.models import User
|
||||
from app.reader import queries, service
|
||||
from app.sync.playlists import (
|
||||
PlaylistGone,
|
||||
lock_playlists,
|
||||
merge_playlist,
|
||||
plan_merge,
|
||||
recompute_dirty,
|
||||
pull_user_playlists,
|
||||
repull_playlist,
|
||||
sync_claim,
|
||||
user_sync_claim,
|
||||
)
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.routes._common import not_found, youtube_refusal
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.reader.models import Playlist
|
||||
|
||||
router = APIRouter(prefix="/api/playlists", tags=["playlists"])
|
||||
|
||||
# SEC-14, same reasoning as the sync module's limiter: a playlist pull lists every playlist the
|
||||
@@ -56,90 +59,20 @@ def _refused(status: int, reason: str) -> HTTPException:
|
||||
return HTTPException(status_code=status, detail={"code": "playlist", "reason": reason})
|
||||
|
||||
|
||||
def _touch(pl: Playlist) -> None:
|
||||
"""Bump the playlist's `updated_at` for an edit that only writes `playlist_items`.
|
||||
|
||||
`UpdatedAtMixin`'s `onupdate` fires on an UPDATE of the PLAYLIST row, and add / remove /
|
||||
reorder / move write the item rows instead — so without this the rail's "recently changed"
|
||||
sort would only ever see renames. The assignment is what makes the column mean "last time
|
||||
this list changed", which is what the rail promises the reader.
|
||||
|
||||
The DATABASE's clock, like the mixin's own `onupdate`: one sort key written from two clocks
|
||||
(this process's and Postgres's) could order a rename and an item edit the wrong way round."""
|
||||
pl.updated_at = func.now()
|
||||
def _own_playlist(db: Session, user: User, playlist_id: int, *, lock: bool = False) -> "Playlist":
|
||||
pl = queries.own_playlist(db, user, playlist_id, lock=lock)
|
||||
if pl is None:
|
||||
raise not_found("playlist")
|
||||
return pl
|
||||
|
||||
|
||||
def _next_playlist_position(db: Session, user_id: int) -> int:
|
||||
"""Append position for a new playlist in this user's rail."""
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.max(Playlist.position), -1)).where(
|
||||
Playlist.user_id == user_id
|
||||
)
|
||||
)
|
||||
+ 1
|
||||
)
|
||||
|
||||
|
||||
def _next_item_position(db: Session, playlist_id: int) -> int:
|
||||
"""Append position for a new item at the end of a playlist. Locks the playlist row first: two
|
||||
adds (two tabs, the rail and the page) otherwise both read the same max and land on one
|
||||
position, and the list's order between them is then arbitrary."""
|
||||
lock_playlists(db, playlist_id)
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
|
||||
PlaylistItem.playlist_id == playlist_id
|
||||
)
|
||||
)
|
||||
+ 1
|
||||
)
|
||||
|
||||
|
||||
def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Append a video to a playlist if not already present (idempotent). Returns whether it
|
||||
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later).
|
||||
Locked BEFORE the presence check: two adds of one video both saw "absent" and the second one's
|
||||
insert hit uq_playlist_video as a 500."""
|
||||
lock_playlists(db, pl.id)
|
||||
exists = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||
)
|
||||
)
|
||||
if exists is not None:
|
||||
return False
|
||||
db.add(
|
||||
PlaylistItem(playlist_id=pl.id, video_id=video_id, position=_next_item_position(db, pl.id))
|
||||
)
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
return True
|
||||
|
||||
|
||||
def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Remove a video from a playlist if present (idempotent). Returns whether it was removed.
|
||||
`mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
||||
lock_playlists(db, pl.id)
|
||||
item = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||
)
|
||||
)
|
||||
if item is None:
|
||||
return False
|
||||
db.delete(item)
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
return True
|
||||
def _video_or_404(db: Session, video_id: str) -> None:
|
||||
if queries.video(db, video_id) is None:
|
||||
raise not_found("video")
|
||||
|
||||
|
||||
def _summary(
|
||||
pl: Playlist,
|
||||
pl: "Playlist",
|
||||
count: int,
|
||||
total_duration: int = 0,
|
||||
has_video: bool | None = None,
|
||||
@@ -156,7 +89,7 @@ def _summary(
|
||||
"total_duration_seconds": total_duration,
|
||||
"cover_thumbnail": cover,
|
||||
# When this list last CHANGED (rename, add/remove/move/reorder) — the rail's
|
||||
# "recently changed first" sort reads it. See _touch for why it isn't automatic.
|
||||
# "recently changed first" sort reads it. See `service.touch` for why it isn't automatic.
|
||||
"updated_at": pl.updated_at,
|
||||
}
|
||||
if has_video is not None:
|
||||
@@ -172,73 +105,14 @@ def list_playlists(
|
||||
) -> list[dict]:
|
||||
"""The user's playlists (newest custom order). With `contains=<video_id>`, each entry
|
||||
also reports whether that video is already in it — used by the add-to-playlist popover."""
|
||||
pls = (
|
||||
db.execute(
|
||||
select(Playlist)
|
||||
.where(Playlist.user_id == user.id)
|
||||
.order_by(Playlist.position, Playlist.created_at)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
pls = queries.playlists_for(db, user)
|
||||
if not pls:
|
||||
return []
|
||||
ids = [p.id for p in pls]
|
||||
counts = dict(
|
||||
db.execute(
|
||||
select(PlaylistItem.playlist_id, func.count())
|
||||
.where(PlaylistItem.playlist_id.in_(ids))
|
||||
.group_by(PlaylistItem.playlist_id)
|
||||
).all()
|
||||
)
|
||||
durations = dict(
|
||||
db.execute(
|
||||
select(
|
||||
PlaylistItem.playlist_id,
|
||||
func.coalesce(func.sum(Video.duration_seconds), 0),
|
||||
)
|
||||
.join(Video, Video.id == PlaylistItem.video_id)
|
||||
.where(PlaylistItem.playlist_id.in_(ids))
|
||||
.group_by(PlaylistItem.playlist_id)
|
||||
).all()
|
||||
)
|
||||
member: set[int] = set()
|
||||
if contains:
|
||||
member = set(
|
||||
db.execute(
|
||||
select(PlaylistItem.playlist_id).where(
|
||||
PlaylistItem.playlist_id.in_(ids),
|
||||
PlaylistItem.video_id == contains,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
# Batch the cover thumbnails (first item per playlist by position) in ONE window query
|
||||
# instead of a per-playlist SELECT inside _summary (the old N+1).
|
||||
rn = (
|
||||
func.row_number()
|
||||
.over(
|
||||
partition_by=PlaylistItem.playlist_id,
|
||||
order_by=PlaylistItem.position,
|
||||
)
|
||||
.label("rn")
|
||||
)
|
||||
first_items = (
|
||||
select(
|
||||
PlaylistItem.playlist_id.label("pid"),
|
||||
Video.thumbnail_url.label("thumb"),
|
||||
rn,
|
||||
)
|
||||
.join(Video, Video.id == PlaylistItem.video_id)
|
||||
.where(PlaylistItem.playlist_id.in_(ids))
|
||||
.subquery()
|
||||
)
|
||||
covers = dict(
|
||||
db.execute(
|
||||
select(first_items.c.pid, first_items.c.thumb).where(first_items.c.rn == 1)
|
||||
).all()
|
||||
)
|
||||
counts = queries.item_counts(db, ids)
|
||||
durations = queries.durations(db, ids)
|
||||
member = queries.containing(db, ids, contains) if contains else set()
|
||||
covers = queries.covers(db, ids)
|
||||
return [
|
||||
_summary(
|
||||
p,
|
||||
@@ -269,29 +143,11 @@ def create_playlist(
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise _refused(400, "name_required")
|
||||
pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id))
|
||||
db.add(pl)
|
||||
pl = service.create_playlist(db, user, name)
|
||||
db.commit()
|
||||
return _summary(pl, 0)
|
||||
|
||||
|
||||
def _watch_later(db: Session, user: User) -> Playlist:
|
||||
"""The user's built-in Watch later playlist, created on first use."""
|
||||
pl = db.scalar(
|
||||
select(Playlist).where(Playlist.user_id == user.id, Playlist.kind == "watch_later")
|
||||
)
|
||||
if pl is None:
|
||||
pl = Playlist(
|
||||
user_id=user.id,
|
||||
name="Watch later",
|
||||
kind="watch_later",
|
||||
position=_next_playlist_position(db, user.id),
|
||||
)
|
||||
db.add(pl)
|
||||
db.flush() # the id, for the item the caller adds next; the handler commits
|
||||
return pl
|
||||
|
||||
|
||||
@router.post("/watch-later")
|
||||
def add_watch_later(
|
||||
body: VideoRefIn,
|
||||
@@ -300,9 +156,9 @@ def add_watch_later(
|
||||
) -> dict:
|
||||
"""Bookmark shortcut: add a video to the built-in Watch later playlist."""
|
||||
video_id = body.video_id
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
pl = _watch_later(db, user)
|
||||
_add_item(db, pl, video_id, mark_dirty=False)
|
||||
_video_or_404(db, video_id)
|
||||
pl = service.watch_later(db, user)
|
||||
service.add_item(db, pl, video_id, mark_dirty=False)
|
||||
db.commit()
|
||||
return {"ok": True, "saved": True, "playlist_id": pl.id}
|
||||
|
||||
@@ -313,11 +169,9 @@ def remove_watch_later(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = db.scalar(
|
||||
select(Playlist).where(Playlist.user_id == user.id, Playlist.kind == "watch_later")
|
||||
)
|
||||
pl = queries.watch_later(db, user)
|
||||
if pl is not None:
|
||||
_remove_item(db, pl, video_id, mark_dirty=False)
|
||||
service.remove_item(db, pl, video_id, mark_dirty=False)
|
||||
db.commit()
|
||||
return {"ok": True, "saved": False}
|
||||
|
||||
@@ -354,7 +208,7 @@ def revert_youtube(
|
||||
"""Reset one YouTube-linked playlist to its current YouTube state, discarding local
|
||||
edits (and clearing the dirty flag). The user-facing 'undo all my changes' for a list
|
||||
once the in-session undo history is gone."""
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
if not pl.yt_playlist_id:
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_read_scope(user):
|
||||
@@ -375,10 +229,10 @@ def revert_youtube(
|
||||
raise youtube_refusal(exc, playlist=True) from exc
|
||||
|
||||
|
||||
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
def _pushable(db: Session, user: User, playlist_id: int) -> "Playlist":
|
||||
"""A local playlist the user may push to YouTube (create or update). The built-in
|
||||
Watch later and read-only YouTube mirrors are never pushable."""
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
if pl.kind == "watch_later":
|
||||
raise _refused(400, "watch_later_not_pushable")
|
||||
return pl
|
||||
@@ -448,17 +302,8 @@ def get_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
state = aliased(VideoState)
|
||||
rows = db.execute(
|
||||
select(*feed_columns(user, state))
|
||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id))
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
# `id` breaks a position tie the same way on every read (the renumbering add does too).
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).all()
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
rows = queries.playlist_rows(db, user, pl.id)
|
||||
return {
|
||||
"id": pl.id,
|
||||
"name": pl.name,
|
||||
@@ -482,34 +327,15 @@ def rename_playlist(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# Locked: the new name moves `dirty`, which the scheduled mirror refresh reads under this lock.
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||
pl = _own_playlist(db, user, playlist_id, lock=True)
|
||||
if "name" in body.model_fields_set:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise _refused(400, "name_required")
|
||||
if name != pl.name:
|
||||
pl.name = name
|
||||
recompute_dirty(db, pl)
|
||||
service.rename_playlist(db, pl, name)
|
||||
db.commit()
|
||||
# One aggregate for count + duration (outer join so an empty playlist still returns 0),
|
||||
# plus a single cover lookup — was 3 separate round-trips.
|
||||
count, total = db.execute(
|
||||
select(
|
||||
func.count(PlaylistItem.id),
|
||||
func.coalesce(func.sum(Video.duration_seconds), 0),
|
||||
)
|
||||
.select_from(PlaylistItem)
|
||||
.outerjoin(Video, Video.id == PlaylistItem.video_id)
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
).one()
|
||||
cover = db.scalar(
|
||||
select(Video.thumbnail_url)
|
||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
.order_by(PlaylistItem.position)
|
||||
.limit(1)
|
||||
)
|
||||
return _summary(pl, count or 0, total or 0, cover=cover)
|
||||
count, total, cover = queries.playlist_totals(db, pl.id)
|
||||
return _summary(pl, count, total, cover=cover)
|
||||
|
||||
|
||||
@router.delete("/{playlist_id}")
|
||||
@@ -519,7 +345,7 @@ def delete_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
if pl.kind == "watch_later":
|
||||
raise _refused(400, "watch_later_undeletable")
|
||||
# Not in the middle of a sync of it (`sync_claim`): a push that read the list before the delete
|
||||
@@ -540,9 +366,7 @@ def delete_playlist(
|
||||
yt.delete_playlist(pl.yt_playlist_id)
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc, playlist=True) from exc
|
||||
# The playlist first, as every item write: the cascade deletes the items before the list row.
|
||||
lock_playlists(db, pl.id)
|
||||
db.delete(pl) # items cascade
|
||||
service.delete_playlist(db, pl)
|
||||
db.commit()
|
||||
return {"deleted": playlist_id}
|
||||
|
||||
@@ -561,26 +385,10 @@ def add_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
video_id = body.video_id
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
added = _add_item(db, pl, video_id, mark_dirty=False)
|
||||
if added and body.position is not None:
|
||||
# Renumber the whole list with the new row at `position` (clamped to the end): positions
|
||||
# can have gaps after removals, so "index N" is only meaningful over the ordered rows.
|
||||
rows = list(
|
||||
db.execute(
|
||||
select(PlaylistItem)
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).scalars()
|
||||
)
|
||||
new = next(r for r in rows if r.video_id == video_id)
|
||||
rows.remove(new)
|
||||
rows.insert(min(body.position, len(rows)), new)
|
||||
for pos, row in enumerate(rows):
|
||||
row.position = pos
|
||||
recompute_dirty(db, pl)
|
||||
_video_or_404(db, video_id)
|
||||
service.add_to_playlist(db, pl, video_id, body.position)
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
@@ -592,8 +400,8 @@ def remove_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
_remove_item(db, pl, video_id, mark_dirty=True)
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
service.remove_item(db, pl, video_id, mark_dirty=True)
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
@@ -617,49 +425,21 @@ def move_item(
|
||||
exactly the kind of half-applied state this codebase treats as a bug rather than an edge
|
||||
case. Idempotent: if the video is already in the target, the source row is simply dropped
|
||||
(moving into a list that already has it is a removal, not a duplicate)."""
|
||||
src = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
src = _own_playlist(db, user, playlist_id)
|
||||
# Its own reason: "this playlist doesn't exist any more" about a TARGET that is gone was read as
|
||||
# the list on screen — which is still there.
|
||||
dst = owned_or_404(db, Playlist, body.target_playlist_id, user, what="move_target")
|
||||
if src.id == dst.id:
|
||||
dst = queries.own_playlist(db, user, body.target_playlist_id)
|
||||
if dst is None:
|
||||
raise not_found("move_target")
|
||||
try:
|
||||
service.move_item(db, src, dst, video_id)
|
||||
except service.MoveRefused as e:
|
||||
# A same-list move is a bad request; a video a concurrent edit (another tab, a YouTube
|
||||
# re-pull) already took out is gone.
|
||||
status = 400 if e.reason == "same_playlist" else 404
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "playlist_move", "reason": "same_playlist"}
|
||||
)
|
||||
# Both lists, in id order: a move A→B beside a move B→A would otherwise lock crosswise.
|
||||
lock_playlists(db, src.id, dst.id)
|
||||
item = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == src.id, PlaylistItem.video_id == video_id
|
||||
)
|
||||
)
|
||||
if item is None:
|
||||
# A concurrent edit (another tab, or a YouTube re-pull) already took it out.
|
||||
raise HTTPException(
|
||||
status_code=404, detail={"code": "playlist_move", "reason": "not_in_playlist"}
|
||||
)
|
||||
already = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == dst.id, PlaylistItem.video_id == video_id
|
||||
)
|
||||
)
|
||||
dst_changed = False
|
||||
if already is None:
|
||||
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
|
||||
# survives the move, and there is no window where the video belongs to neither list. The
|
||||
# check above holds: both lists are locked, and an insert into the target (another tab's
|
||||
# add, a YouTube merge) waits for the lock — its foreign key needs the target row.
|
||||
item.playlist_id = dst.id
|
||||
item.position = _next_item_position(db, dst.id)
|
||||
dst_changed = True
|
||||
if not dst_changed:
|
||||
db.delete(item)
|
||||
# The source always changed; the target only when the row landed in it. `_touch` on a target
|
||||
# that already had the video promoted an untouched list in the rail's "recently changed" sort.
|
||||
recompute_dirty(db, src)
|
||||
_touch(src)
|
||||
if dst_changed:
|
||||
recompute_dirty(db, dst)
|
||||
_touch(dst)
|
||||
status_code=status, detail={"code": "playlist_move", "reason": e.reason}
|
||||
) from None
|
||||
db.commit()
|
||||
return {"ok": True, "from_playlist_id": src.id, "to_playlist_id": dst.id, "video_id": video_id}
|
||||
|
||||
@@ -676,30 +456,7 @@ def reorder_items(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# Locked: a concurrent add between reading the items and writing the order would collide.
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||
order = body.video_ids
|
||||
items = {
|
||||
it.video_id: it
|
||||
for it in db.execute(
|
||||
select(PlaylistItem).where(PlaylistItem.playlist_id == pl.id)
|
||||
).scalars()
|
||||
}
|
||||
pos = 0
|
||||
seen: set[str] = set()
|
||||
for vid in order:
|
||||
it = items.get(vid)
|
||||
if it is not None:
|
||||
it.position = pos
|
||||
pos += 1
|
||||
seen.add(vid)
|
||||
# Any items the payload omitted (e.g. added concurrently in another tab) keep their relative
|
||||
# order but get fresh trailing positions, so no two items collide on the same position.
|
||||
for it in sorted(
|
||||
(it for vid, it in items.items() if vid not in seen), key=lambda i: i.position
|
||||
):
|
||||
it.position = pos
|
||||
pos += 1
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
pl = _own_playlist(db, user, playlist_id, lock=True)
|
||||
pos = service.reorder_items(db, pl, body.video_ids)
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "count": pos}
|
||||
|
||||
@@ -6,23 +6,26 @@ the app or any account data. Password-protected links exchange the password once
|
||||
a short-lived signed grant that the media URL carries (a `<video src>` can't send a header).
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import storage
|
||||
from app.downloads import queries, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset, User
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import verify_password
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.downloads.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
|
||||
router = APIRouter(prefix="/api/public", tags=["public"])
|
||||
|
||||
# Slow password guessing on a link (the token itself is 192-bit, so it needs no throttle). Counted per
|
||||
@@ -37,26 +40,25 @@ _unlock_limit = RateLimiter(max_events=10, window_seconds=300)
|
||||
_unlock_failures = RateLimiter(max_events=50, window_seconds=300)
|
||||
|
||||
|
||||
def _link_or_404(db: Session, token: str) -> DownloadLink:
|
||||
def _link_or_404(db: Session, token: str) -> "DownloadLink":
|
||||
link = linksmod.find(db, token)
|
||||
if link is None or linksmod.is_expired(link):
|
||||
raise refused(404, "link_gone")
|
||||
return link
|
||||
|
||||
|
||||
def _being_made(db: Session, job: DownloadJob) -> bool:
|
||||
def _being_made(db: Session, job: "DownloadJob") -> bool:
|
||||
"""The job is running, or queued for a worker that will take it: the worker leaves a suspended
|
||||
owner's queued jobs where they are (`worker._claim_job`), as long as the suspension lasts."""
|
||||
if job.status == JobStatus.RUNNING:
|
||||
return True
|
||||
if job.status != JobStatus.QUEUED:
|
||||
return False
|
||||
owner = db.get(User, job.user_id)
|
||||
return owner is not None and not owner.is_suspended
|
||||
return not queries.owner_suspended(db, job)
|
||||
|
||||
|
||||
def _ready_job(db: Session, job: DownloadJob | None) -> tuple[DownloadJob, MediaAsset]:
|
||||
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
||||
def _ready_job(db: Session, job: "DownloadJob | None") -> tuple["DownloadJob", "MediaAsset"]:
|
||||
asset = queries.asset_of(db, job)
|
||||
if (
|
||||
job is not None
|
||||
and asset is not None
|
||||
@@ -78,25 +80,24 @@ def _ready_job(db: Session, job: DownloadJob | None) -> tuple[DownloadJob, Media
|
||||
return job, asset
|
||||
|
||||
|
||||
def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAsset]:
|
||||
def _ready_target(db: Session, link: "DownloadLink") -> tuple["DownloadJob", "MediaAsset"]:
|
||||
"""A file link's file. A folder link names no one file: its files are asked for by id."""
|
||||
if link.job_id is None:
|
||||
raise refused(404, "video_gone")
|
||||
return _ready_job(db, db.get(DownloadJob, link.job_id))
|
||||
return _ready_job(db, queries.job(db, link.job_id))
|
||||
|
||||
|
||||
def _link_folder(db: Session, link: DownloadLink) -> FileFolder:
|
||||
folder = db.get(FileFolder, link.folder_id) if link.folder_id is not None else None
|
||||
def _link_folder(db: Session, link: "DownloadLink") -> "FileFolder":
|
||||
folder = queries.link_folder(db, link)
|
||||
if folder is None:
|
||||
raise refused(404, "link_gone")
|
||||
return folder
|
||||
|
||||
|
||||
def _folder_job(db: Session, link: DownloadLink, job_id: int) -> tuple[DownloadJob, MediaAsset]:
|
||||
folder = _link_folder(db, link)
|
||||
job = db.execute(
|
||||
select(DownloadJob).where(DownloadJob.id == job_id, *linksmod.in_folder(folder))
|
||||
).scalar_one_or_none()
|
||||
def _folder_job(
|
||||
db: Session, link: "DownloadLink", job_id: int
|
||||
) -> tuple["DownloadJob", "MediaAsset"]:
|
||||
job = queries.folder_job(db, _link_folder(db, link), job_id)
|
||||
if job is None:
|
||||
raise refused(404, "video_gone")
|
||||
return _ready_job(db, job)
|
||||
@@ -113,7 +114,7 @@ def _file_url(token: str, grant: str | None = None, job_id: int | None = None) -
|
||||
|
||||
|
||||
def _poster_url(
|
||||
token: str, asset: MediaAsset, grant: str | None = None, job_id: int | None = None
|
||||
token: str, asset: "MediaAsset", grant: str | None = None, job_id: int | None = None
|
||||
) -> str | None:
|
||||
if not asset.poster_path:
|
||||
return None
|
||||
@@ -121,7 +122,7 @@ def _poster_url(
|
||||
return f"{url}?g={grant}" if grant else url
|
||||
|
||||
|
||||
def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict:
|
||||
def _folder_meta(db: Session, link: "DownloadLink", token: str, grant: str | None) -> dict:
|
||||
"""A folder link's page: the folder's name and each file's own watch metadata
|
||||
(`linksmod.folder_files`: ready, on disk, newest first)."""
|
||||
folder = _link_folder(db, link)
|
||||
@@ -147,7 +148,7 @@ def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None)
|
||||
}
|
||||
|
||||
|
||||
def _file_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict:
|
||||
def _file_meta(db: Session, link: "DownloadLink", token: str, grant: str | None) -> dict:
|
||||
job, asset = _ready_target(db, link)
|
||||
return {
|
||||
"needs_password": False,
|
||||
@@ -158,20 +159,14 @@ def _file_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -
|
||||
}
|
||||
|
||||
|
||||
def _opened(db: Session, link: DownloadLink, token: str, grant: str | None = None) -> dict:
|
||||
def _opened(db: Session, link: "DownloadLink", token: str, grant: str | None = None) -> dict:
|
||||
"""The page an open link shows, counted as one view (the handler commits the count)."""
|
||||
meta = (
|
||||
_folder_meta(db, link, token, grant)
|
||||
if link.folder_id is not None
|
||||
else _file_meta(db, link, token, grant)
|
||||
)
|
||||
# In SQL, not `+= 1` on the loaded row: two opens at once would both write the same count.
|
||||
db.execute(
|
||||
update(DownloadLink)
|
||||
.where(DownloadLink.id == link.id)
|
||||
.values(view_count=DownloadLink.view_count + 1)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
linksmod.count_view(db, link)
|
||||
return meta
|
||||
|
||||
|
||||
@@ -217,13 +212,13 @@ def watch_unlock(
|
||||
return {"ok": True, **meta}
|
||||
|
||||
|
||||
def _granted(link: DownloadLink, token: str, g: str | None) -> None:
|
||||
def _granted(link: "DownloadLink", token: str, g: str | None) -> None:
|
||||
"""A password link's files want the grant its unlock handed out."""
|
||||
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
||||
raise refused(403, "share_password_required")
|
||||
|
||||
|
||||
def _serve(link: DownloadLink, asset: MediaAsset) -> FileResponse:
|
||||
def _serve(link: "DownloadLink", asset: "MediaAsset") -> FileResponse:
|
||||
"""Range-aware stream of a shared file. Inline by default (plays in the page); an
|
||||
`allow_download` link serves it as an attachment."""
|
||||
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||
@@ -252,7 +247,7 @@ def _serve(link: DownloadLink, asset: MediaAsset) -> FileResponse:
|
||||
)
|
||||
|
||||
|
||||
def _poster(asset: MediaAsset) -> FileResponse:
|
||||
def _poster(asset: "MediaAsset") -> FileResponse:
|
||||
if not asset.poster_path:
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||
|
||||
@@ -6,7 +6,7 @@ The aggregate daily budget lives in ApiQuotaUsage; here we read the per-spend au
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -14,6 +14,8 @@ from app import quota
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import QuotaEvent, User
|
||||
from app.routes import _account as account
|
||||
from app.routes._keyset import utc
|
||||
|
||||
router = APIRouter(prefix="/api/quota", tags=["quota"])
|
||||
|
||||
@@ -45,23 +47,32 @@ def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db))
|
||||
|
||||
@router.get("/admin")
|
||||
def admin_usage(
|
||||
days: int = 30,
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
user: str | None = Query(default=None, pattern=account.PATTERN),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
days = max(1, min(days, 365))
|
||||
since = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
"""Spend per account (and per action) and per day, between `since` (default: 30 days back) and
|
||||
`until` (default: now), optionally for one account (`none` = the background)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
lo = utc(since) or now - timedelta(days=30)
|
||||
window = [QuotaEvent.created_at >= lo]
|
||||
if until is not None:
|
||||
window.append(QuotaEvent.created_at < utc(until))
|
||||
if user is not None:
|
||||
window.append(account.condition(QuotaEvent.user_id, user))
|
||||
|
||||
totals = db.execute(
|
||||
select(QuotaEvent.user_id, func.sum(QuotaEvent.units))
|
||||
.where(QuotaEvent.created_at >= since)
|
||||
.where(*window)
|
||||
.group_by(QuotaEvent.user_id)
|
||||
).all()
|
||||
|
||||
per_action: dict[int | None, dict[str, int]] = {}
|
||||
for uid, action, units in db.execute(
|
||||
select(QuotaEvent.user_id, QuotaEvent.action, func.sum(QuotaEvent.units))
|
||||
.where(QuotaEvent.created_at >= since)
|
||||
.where(*window)
|
||||
.group_by(QuotaEvent.user_id, QuotaEvent.action)
|
||||
).all():
|
||||
per_action.setdefault(uid, {})[action] = int(units)
|
||||
@@ -84,16 +95,16 @@ def admin_usage(
|
||||
]
|
||||
rows.sort(key=lambda r: r["total"], reverse=True)
|
||||
|
||||
# Instance-wide daily series in Pacific days (matches the budget reset).
|
||||
# The daily series in Pacific days (matches the budget reset), of the one account when filtered.
|
||||
day_expr = func.date(func.timezone("America/Los_Angeles", QuotaEvent.created_at))
|
||||
daily = [
|
||||
{"day": str(day), "total": int(units)}
|
||||
for day, units in db.execute(
|
||||
select(day_expr, func.sum(QuotaEvent.units))
|
||||
.where(QuotaEvent.created_at >= since)
|
||||
.where(*window)
|
||||
.group_by(day_expr)
|
||||
.order_by(day_expr)
|
||||
).all()
|
||||
]
|
||||
|
||||
return {"range_days": days, "rows": rows, "daily": daily}
|
||||
return {"rows": rows, "daily": daily}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Admin background-run history: read `job_runs`, a page at a time.
|
||||
|
||||
Admin-only and read-only — rows leave the table through the retention job
|
||||
(`job_run_retention_days`). Filtered on the server and paged by `_keyset`, like the server log and
|
||||
the audit log beside it on the activity page. A row names what a reader needs without a second
|
||||
request: the channel a user's ad-hoc run worked on, and how many log lines the run left (only
|
||||
WARNING and up are stored, so most runs have none — the count says so before the click).
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import scheduler
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import Channel, JobRun, LogEvent, User
|
||||
from app.routes import _account as account
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.runstatus import RunStatus
|
||||
from app.utils import iso, like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/runs", tags=["admin"])
|
||||
|
||||
Facet = Literal["job", "status"]
|
||||
|
||||
|
||||
def _serialize(run: JobRun, titles: dict[str, str], lines: dict[int, int]) -> dict:
|
||||
return {
|
||||
"id": run.id,
|
||||
"job": run.job,
|
||||
"trigger": run.trigger,
|
||||
"actor_id": run.actor_id,
|
||||
"target": run.target,
|
||||
"target_title": titles.get(run.target) if run.target is not None else None,
|
||||
"status": run.status,
|
||||
"started_at": iso(run.started_at),
|
||||
"finished_at": iso(run.finished_at),
|
||||
"summary": run.summary,
|
||||
"result": run.result,
|
||||
"error": run.error,
|
||||
"failure": run.failure,
|
||||
"quota_units": run.quota_units,
|
||||
"log_lines": lines.get(run.id, 0),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunFilters:
|
||||
job: str | None
|
||||
status: RunStatus | None
|
||||
actor: str | None
|
||||
q: str | None
|
||||
since: datetime | None
|
||||
until: datetime | None
|
||||
|
||||
def conditions(self, skip: Facet | None = None) -> list:
|
||||
"""The WHERE conditions, ANDed. `skip` leaves one menu's own filter out, so that menu counts
|
||||
its values under every other filter (the way back from a pick)."""
|
||||
out = []
|
||||
if self.job is not None and skip != "job":
|
||||
out.append(JobRun.job == self.job)
|
||||
if self.status is not None and skip != "status":
|
||||
out.append(JobRun.status == self.status)
|
||||
if self.actor is not None:
|
||||
out.append(account.condition(JobRun.actor_id, self.actor))
|
||||
if self.since is not None:
|
||||
out.append(JobRun.started_at >= utc(self.since))
|
||||
if self.until is not None:
|
||||
out.append(JobRun.started_at < utc(self.until))
|
||||
if self.q and self.q.strip():
|
||||
like = f"%{like_escape(self.q.strip())}%"
|
||||
out.append(
|
||||
or_(
|
||||
JobRun.job.ilike(like),
|
||||
JobRun.target.ilike(like),
|
||||
JobRun.summary.ilike(like),
|
||||
JobRun.error.ilike(like),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def run_filters(
|
||||
job: str | None = Query(default=None, max_length=40, pattern=r"^[a-z][a-z0-9_]*$"),
|
||||
status: RunStatus | None = None,
|
||||
user: str | None = Query(default=None, pattern=account.PATTERN),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> RunFilters:
|
||||
return RunFilters(job, status, user, q, since, until)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_runs(
|
||||
filters: RunFilters = Depends(run_filters),
|
||||
before: str | None = Query(default=None, max_length=200),
|
||||
limit: int = Query(default=100, ge=1, le=PAGE_MAX),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""One page of runs, newest first, matching every filter given; `before` continues from a
|
||||
previous page's `next_cursor`."""
|
||||
query = select(JobRun).where(*filters.conditions())
|
||||
if before is not None:
|
||||
query = query.where(older_than(JobRun.started_at, JobRun.id, before))
|
||||
order = (JobRun.started_at.desc(), JobRun.id.desc())
|
||||
runs, cursor = page(
|
||||
list(db.execute(query.order_by(*order).limit(limit + 1)).scalars().all()),
|
||||
limit,
|
||||
lambda r: r.started_at,
|
||||
)
|
||||
channels = {r.target for r in runs if r.job in scheduler.AD_HOC_JOBS and r.target}
|
||||
titles: dict[str, str] = {}
|
||||
if channels:
|
||||
rows = db.execute(select(Channel.id, Channel.title).where(Channel.id.in_(channels)))
|
||||
titles = {cid: title for cid, title in rows.tuples() if title is not None}
|
||||
lines: dict[int, int] = {}
|
||||
if runs:
|
||||
counted = db.execute(
|
||||
select(LogEvent.run_id, func.count())
|
||||
.where(LogEvent.run_id.in_([r.id for r in runs]))
|
||||
.group_by(LogEvent.run_id)
|
||||
)
|
||||
lines = {rid: n for rid, n in counted.tuples() if rid is not None}
|
||||
return {
|
||||
"items": [_serialize(r, titles, lines) for r in runs],
|
||||
"accounts": accounts(db, {r.actor_id for r in runs if r.actor_id is not None}),
|
||||
"next_cursor": cursor,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/facets")
|
||||
def run_facets(
|
||||
filters: RunFilters = Depends(run_filters),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""What the job menu and the status chips offer, each counted under every filter but its own."""
|
||||
|
||||
def counts(col, skip: Facet) -> list[dict]:
|
||||
n = func.count().label("n")
|
||||
stmt = select(col, n).where(*filters.conditions(skip)).group_by(col).order_by(n.desc(), col)
|
||||
return [{"value": v, "count": c} for v, c in db.execute(stmt).all()]
|
||||
|
||||
total = select(func.count()).select_from(JobRun).where(*filters.conditions("status"))
|
||||
return {
|
||||
"jobs": counts(JobRun.job, "job"),
|
||||
"statuses": counts(JobRun.status, "status"),
|
||||
"total": db.scalar(total) or 0,
|
||||
}
|
||||
@@ -1,15 +1,20 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel, field_validator
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import SavedView, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.models import User
|
||||
from app.reader import queries, service
|
||||
from app.routes._common import not_found
|
||||
from app.refusal import refused
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.reader.models import SavedView
|
||||
|
||||
# Saved "smart views": per-user named snapshots of a module's filters (feed or plex, keyed by
|
||||
# `source`).
|
||||
router = APIRouter(prefix="/api/saved-views", tags=["saved-views"])
|
||||
@@ -23,7 +28,7 @@ def _norm_source(source: str) -> str:
|
||||
return source if source in _SOURCES else "feed"
|
||||
|
||||
|
||||
def _serialize(v: SavedView) -> dict:
|
||||
def _serialize(v: "SavedView") -> dict:
|
||||
return {
|
||||
"id": v.id,
|
||||
"name": v.name,
|
||||
@@ -33,23 +38,20 @@ def _serialize(v: SavedView) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _own_view(db: Session, user: User, view_id: int) -> "SavedView":
|
||||
view = queries.own_view(db, user, view_id)
|
||||
if view is None:
|
||||
raise not_found("view")
|
||||
return view
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_views(
|
||||
source: str = "feed",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> list[dict]:
|
||||
source = _norm_source(source)
|
||||
rows = (
|
||||
db.execute(
|
||||
select(SavedView)
|
||||
.where(SavedView.user_id == user.id, SavedView.source == source)
|
||||
.order_by(SavedView.position, SavedView.id)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return [_serialize(v) for v in rows]
|
||||
return [_serialize(v) for v in queries.views(db, user, _norm_source(source))]
|
||||
|
||||
|
||||
# A feed view's tag pick is read by more than the feed: the autotag job keeps every tag a view names
|
||||
@@ -86,22 +88,8 @@ def create_view(
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise refused(400, "name_required")
|
||||
source = _norm_source(body.source)
|
||||
# Position is per (user, source), so each module's views number 0..N independently.
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(SavedView.position), -1)).where(
|
||||
SavedView.user_id == user.id, SavedView.source == source
|
||||
)
|
||||
).scalar_one()
|
||||
view = SavedView(
|
||||
user_id=user.id,
|
||||
source=source,
|
||||
name=name[:80],
|
||||
filters=body.filters,
|
||||
position=maxpos + 1,
|
||||
)
|
||||
db.add(view)
|
||||
try:
|
||||
view = service.create_view(db, user, _norm_source(body.source), name, body.filters)
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
@@ -126,32 +114,17 @@ def update_view(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
view = owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
view = _own_view(db, user, view_id)
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise refused(400, "name_required")
|
||||
view.name = name[:80]
|
||||
if "filters" in sent:
|
||||
if body.filters is None:
|
||||
raise refused(400, "filters_not_object")
|
||||
view.filters = body.filters
|
||||
if "is_default" in sent:
|
||||
make_default = bool(body.is_default)
|
||||
if make_default:
|
||||
# At most one default per (user, source): clear the flag on this module's others first.
|
||||
db.execute(
|
||||
update(SavedView)
|
||||
.where(
|
||||
SavedView.user_id == user.id,
|
||||
SavedView.source == view.source,
|
||||
SavedView.id != view.id,
|
||||
)
|
||||
.values(is_default=False)
|
||||
)
|
||||
view.is_default = make_default
|
||||
name = (body.name or "").strip()
|
||||
if "name" in sent and not name:
|
||||
raise refused(400, "name_required")
|
||||
if "filters" in sent and body.filters is None:
|
||||
raise refused(400, "filters_not_object")
|
||||
try:
|
||||
service.update_view(
|
||||
db, view, sent=sent, name=name, filters=body.filters, is_default=body.is_default
|
||||
)
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
@@ -165,8 +138,7 @@ def delete_view(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
view = owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
db.delete(view)
|
||||
service.delete_view(db, _own_view(db, user, view_id))
|
||||
db.commit()
|
||||
return {"deleted": view_id}
|
||||
|
||||
@@ -181,14 +153,6 @@ def reorder_views(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
ids = body.ids
|
||||
rows = db.execute(select(SavedView).where(SavedView.user_id == user.id)).scalars().all()
|
||||
by_id = {v.id: v for v in rows}
|
||||
pos = 0
|
||||
for vid in ids:
|
||||
v = by_id.get(vid)
|
||||
if v is not None:
|
||||
v.position = pos
|
||||
pos += 1
|
||||
service.reorder_views(db, user, body.ids)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
@@ -17,8 +17,9 @@ from app.scheduler import (
|
||||
JOB_INTERVALS,
|
||||
MAX_INTERVAL,
|
||||
MIN_INTERVAL,
|
||||
apply_interval,
|
||||
reschedule,
|
||||
scheduler_snapshot,
|
||||
stage_interval,
|
||||
trigger_all,
|
||||
trigger_job,
|
||||
)
|
||||
@@ -42,14 +43,14 @@ def set_job_interval(
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Change how often a scheduler job runs (minutes). Persisted as an override and applied
|
||||
to the live scheduler immediately."""
|
||||
"""Change how often a scheduler job runs (minutes). Stored as an override together with its
|
||||
audit row, then applied to the live scheduler."""
|
||||
if job_id not in JOB_INTERVALS:
|
||||
raise not_found("job")
|
||||
minutes = body.interval_minutes
|
||||
if not (MIN_INTERVAL <= minutes <= MAX_INTERVAL):
|
||||
raise refused(400, "invalid_interval", min=MIN_INTERVAL, max=MAX_INTERVAL)
|
||||
applied = apply_interval(db, job_id, minutes)
|
||||
applied = stage_interval(db, job_id, minutes)
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
@@ -60,6 +61,7 @@ def set_job_interval(
|
||||
after={"interval_minutes": applied},
|
||||
)
|
||||
db.commit()
|
||||
reschedule(job_id, applied)
|
||||
return {"ok": True, "id": job_id, "interval_minutes": applied}
|
||||
|
||||
|
||||
@@ -117,7 +119,7 @@ def set_maintenance_settings(
|
||||
|
||||
@router.get("")
|
||||
def get_scheduler(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
snap = scheduler_snapshot()
|
||||
snap = scheduler_snapshot(db)
|
||||
|
||||
# Work still queued (shared catalog, so these are instance-wide, not per-user).
|
||||
deep_requested = (
|
||||
|
||||
@@ -29,14 +29,12 @@ from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SearchFind,
|
||||
Subscription,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.reader.models import Playlist, PlaylistItem, VideoState
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.utils import like_escape
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
|
||||
@@ -91,12 +91,12 @@ def setup_admin(body: SetupAdminIn, db: Session = Depends(require_setup)) -> dic
|
||||
|
||||
@router.post("/config")
|
||||
def setup_config(settings_map: dict[str, Any], db: Session = Depends(require_setup)) -> dict:
|
||||
"""Persist provided DB-backed settings (Google creds, SMTP, quota…). Keys must be in the
|
||||
sysconfig registry; empty values are skipped (left at the default). Genuinely dynamic
|
||||
"""Persist provided DB-backed settings (Google creds, SMTP, quota…). Keys must be the
|
||||
Configuration page's; empty values are skipped (left at the default). Genuinely dynamic
|
||||
key→value map, so it stays a typed dict rather than a fixed model."""
|
||||
saved: list[str] = []
|
||||
for key, value in settings_map.items():
|
||||
if sysconfig.spec(key) is None:
|
||||
if sysconfig.config_page_spec(key) is None:
|
||||
raise refused(400, "unknown_setting", key=key)
|
||||
if value is None or value == "":
|
||||
continue
|
||||
|
||||
+24
-32
@@ -1,19 +1,32 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import ChannelTag, Tag, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.models import User
|
||||
from app.reader import queries, service
|
||||
from app.routes._common import not_found
|
||||
from app.sync.tagsuggest import compute_tag_suggestions
|
||||
from app.refusal import refused
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.reader.models import Tag
|
||||
|
||||
router = APIRouter(prefix="/api/tags", tags=["tags"])
|
||||
|
||||
|
||||
def _own_tag(db: Session, user: User, tag_id: int) -> "Tag":
|
||||
"""One of the user's own tags, or a 404 — a system tag (no owner) is not theirs to edit."""
|
||||
tag = queries.own_tag(db, user, tag_id)
|
||||
if tag is None:
|
||||
raise not_found("tag")
|
||||
return tag
|
||||
|
||||
|
||||
@router.get("/suggestions")
|
||||
def tag_suggestions(
|
||||
user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
@@ -27,13 +40,6 @@ def tag_suggestions(
|
||||
|
||||
@router.get("")
|
||||
def list_tags(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
rows = db.execute(
|
||||
select(Tag, func.count(ChannelTag.id))
|
||||
.outerjoin(ChannelTag, ChannelTag.tag_id == Tag.id)
|
||||
.where(or_(Tag.user_id.is_(None), Tag.user_id == user.id))
|
||||
.group_by(Tag.id)
|
||||
.order_by(Tag.category, Tag.name)
|
||||
).all()
|
||||
return [
|
||||
{
|
||||
"id": tag.id,
|
||||
@@ -43,7 +49,7 @@ def list_tags(user: User = Depends(current_user), db: Session = Depends(get_db))
|
||||
"system": tag.user_id is None,
|
||||
"channel_count": count,
|
||||
}
|
||||
for tag, count in rows
|
||||
for tag, count in queries.tags_with_counts(db, user)
|
||||
]
|
||||
|
||||
|
||||
@@ -62,16 +68,8 @@ def create_tag(
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise refused(400, "name_required")
|
||||
# An unknown category silently falls back to "other" (unchanged behaviour) rather than 422.
|
||||
category = body.category if body.category in ("language", "topic", "other") else "other"
|
||||
tag = Tag(
|
||||
user_id=user.id,
|
||||
name=name,
|
||||
color=body.color,
|
||||
category=category,
|
||||
)
|
||||
db.add(tag)
|
||||
try:
|
||||
tag = service.create_tag(db, user, name, body.category, body.color)
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
@@ -99,17 +97,13 @@ def update_tag(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# System tags (user_id NULL) are not user-editable: owned_or_404 404s them too.
|
||||
tag = owned_or_404(db, Tag, tag_id, user, what="tag")
|
||||
tag = _own_tag(db, user, tag_id)
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise refused(400, "name_required")
|
||||
tag.name = name
|
||||
if "color" in sent:
|
||||
tag.color = body.color
|
||||
name = (body.name or "").strip()
|
||||
if "name" in sent and not name:
|
||||
raise refused(400, "name_required")
|
||||
try:
|
||||
service.update_tag(tag, sent=sent, name=name, color=body.color)
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
@@ -129,8 +123,6 @@ def delete_tag(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# System tags (user_id NULL) are not user-editable: owned_or_404 404s them too.
|
||||
tag = owned_or_404(db, Tag, tag_id, user, what="tag")
|
||||
db.delete(tag) # ChannelTag rows cascade on tag delete.
|
||||
service.delete_tag(db, _own_tag(db, user, tag_id))
|
||||
db.commit()
|
||||
return {"deleted": tag_id}
|
||||
|
||||
@@ -21,8 +21,8 @@ from app.apitokens import GRANT_COLUMNS, Right
|
||||
from app.audit import AuditAction
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.downloads import folders
|
||||
from app.models import ApiToken, ApiTokenGrant, FileFolder, User
|
||||
from app.downloads import folders, queries
|
||||
from app.models import ApiToken, ApiTokenGrant, User
|
||||
from app.refusal import refused
|
||||
from app.routes._common import owned_or_404
|
||||
from app.security import hash_token
|
||||
@@ -50,12 +50,16 @@ class TokenIn(BaseModel):
|
||||
|
||||
|
||||
def _serialize(db: Session, token: ApiToken) -> dict:
|
||||
rows = db.execute(
|
||||
select(ApiTokenGrant, FileFolder)
|
||||
.join(FileFolder, FileFolder.id == ApiTokenGrant.folder_id)
|
||||
.where(ApiTokenGrant.token_id == token.id)
|
||||
.order_by(FileFolder.id)
|
||||
).all()
|
||||
grants = list(
|
||||
db.execute(
|
||||
select(ApiTokenGrant)
|
||||
.where(ApiTokenGrant.token_id == token.id)
|
||||
.order_by(ApiTokenGrant.folder_id)
|
||||
).scalars()
|
||||
)
|
||||
# The folders are Capture's: read through its queries, not its table.
|
||||
by_id = queries.folders_by_id(db, {g.folder_id for g in grants})
|
||||
rows = [(g, by_id[g.folder_id]) for g in grants if g.folder_id in by_id]
|
||||
now = datetime.now(timezone.utc)
|
||||
return {
|
||||
"id": token.id,
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""The words a background run's row is written with — the `job_runs` table's status and trigger.
|
||||
|
||||
One source for the literals: the scheduler writes them, the startup sweep rewrites `running`, the
|
||||
model's CHECK constraints are built from these sets, and the dashboard reads them back (the frontend's
|
||||
copies are held to them by `backendIds.contract.test.ts`). The migrated database's CHECKs are frozen
|
||||
in its migration: a test holds them equal to these sets, so a new word fails until a migration adds
|
||||
it (`test_runstatus.py`). The download pipeline's `cancelled`/`canceled` drift (`downloads/status.py`)
|
||||
is why a status is never typed as a bare string.
|
||||
"""
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class RunStatus(StrEnum):
|
||||
RUNNING = "running"
|
||||
OK = "ok"
|
||||
ERROR = "error"
|
||||
# Sync was paused: the run was due and deliberately did nothing.
|
||||
SKIPPED = "skipped"
|
||||
# The process stopped while the run was going (a deploy, a crash): set at the next startup, so a
|
||||
# row never claims to be running in a process that no longer exists.
|
||||
INTERRUPTED = "interrupted"
|
||||
|
||||
|
||||
class RunTrigger(StrEnum):
|
||||
# The run's interval came round.
|
||||
SCHEDULED = "scheduled"
|
||||
# An admin pressed "run now" (or "run all").
|
||||
MANUAL = "manual"
|
||||
# A user's own action started work on one target (a channel's full history), outside the
|
||||
# recurring jobs.
|
||||
AD_HOC = "ad_hoc"
|
||||
|
||||
|
||||
def sql_in(values: type[StrEnum]) -> str:
|
||||
"""`'a', 'b'` — the IN-list a CHECK constraint names these values with."""
|
||||
return ", ".join(f"'{v.value}'" for v in values)
|
||||
+346
-190
@@ -3,28 +3,31 @@ recent-first / deep backfill. Runs inside the API process (single worker)."""
|
||||
|
||||
import contextvars
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from datetime import timezone
|
||||
from typing import Callable
|
||||
|
||||
from apscheduler.executors.pool import ThreadPoolExecutor
|
||||
from apscheduler.jobstores.base import ConflictingIdError
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import audit, debugsession, logs, logsink, progress, quota
|
||||
from app.audit import AuditAction
|
||||
from app import audit, debugsession, jobruns, logs, logsink, progress, quota, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads.gc import run_download_gc
|
||||
from app.models import SchedulerSetting
|
||||
from app.models import Channel, JobRun
|
||||
from app.plex.sync import sync as run_plex_sync
|
||||
from app.plex.tmdb import run_enrich as run_tmdb_enrich
|
||||
from app.plex.tmdb import run_people_enrich
|
||||
from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync
|
||||
from app.notifications import create_notification
|
||||
from app.runstatus import RunStatus, RunTrigger
|
||||
from app.state import is_sync_paused
|
||||
from app.sync.autotag import run_autotag_all
|
||||
from app.sync.explore import purge_ephemeral
|
||||
from app.sync.maintenance import run_maintenance
|
||||
from app.sync.playlists import sync_all_playlists
|
||||
from app.utils import iso
|
||||
from app.sync.runner import (
|
||||
run_deep_backfill,
|
||||
run_enrich,
|
||||
@@ -37,12 +40,31 @@ from app.sync.runner import (
|
||||
log = logs.get_logger(__name__)
|
||||
_scheduler: BackgroundScheduler | None = None
|
||||
|
||||
# Per-job run activity, kept in-memory for the admin Scheduler dashboard to read (the
|
||||
# scheduler lives in the same single-worker process as the API, so a request can read it
|
||||
# directly). Ephemeral by design — it reflects "what the scheduler is doing right now and
|
||||
# how the last runs went" since this process started; durable history isn't the goal here.
|
||||
# What is running RIGHT NOW, per job: the single-run claim and the live progress. In memory on
|
||||
# purpose — the scheduler lives in the API's one process, and progress changes every second. What a
|
||||
# run did is durable instead: its `job_runs` row (app/jobruns.py), which outlives a restart.
|
||||
_activity: dict[str, dict] = {}
|
||||
_activity_lock = threading.Lock()
|
||||
# The runs this process has open (run id -> job), under the same lock. A row can stay `running` in
|
||||
# the table after its outcome failed to write (_close), so the table cannot say what is going; this
|
||||
# can, as the one process per database (app/jobruns.py) is the one running them.
|
||||
_open_runs: dict[int, str] = {}
|
||||
|
||||
|
||||
def _opened(run_id: int, job: str) -> None:
|
||||
with _activity_lock:
|
||||
_open_runs[run_id] = job
|
||||
|
||||
|
||||
def _closed(run_id: int) -> None:
|
||||
with _activity_lock:
|
||||
_open_runs.pop(run_id, None)
|
||||
|
||||
|
||||
def _open_run_ids() -> dict[int, str]:
|
||||
with _activity_lock:
|
||||
return dict(_open_runs)
|
||||
|
||||
|
||||
# Set (per-thread) by a manual "run now" trigger to the id of the admin who clicked, so that
|
||||
# run — and only that run, not the recurring scheduled ones — posts a completion notification
|
||||
@@ -51,59 +73,36 @@ _manual_actor: contextvars.ContextVar[int | None] = contextvars.ContextVar(
|
||||
"manual_actor", default=None
|
||||
)
|
||||
|
||||
# Each interval job's env/config default period (minutes). The admin can override any of
|
||||
# these at runtime (stored in scheduler_settings, applied live); see load_intervals.
|
||||
# Each interval job's env/config default period (minutes). The admin can override any of these at
|
||||
# runtime from the Scheduler page — a sysconfig key per job (`sysconfig.interval_key`), applied
|
||||
# live; see load_intervals.
|
||||
JOB_INTERVALS: dict[str, int] = {
|
||||
"rss_poll": settings.rss_poll_minutes,
|
||||
"enrich": settings.enrich_interval_minutes,
|
||||
"backfill": settings.backfill_interval_minutes,
|
||||
"autotag": settings.autotag_interval_minutes,
|
||||
"shorts": settings.shorts_probe_interval_minutes,
|
||||
"subscriptions": settings.subscriptions_resync_minutes,
|
||||
"playlist_sync": settings.playlist_sync_minutes,
|
||||
"maintenance": settings.maintenance_interval_minutes,
|
||||
"explore_cleanup": settings.explore_cleanup_minutes,
|
||||
"download_gc": settings.download_gc_minutes,
|
||||
"plex_sync": settings.plex_sync_interval_min,
|
||||
"plex_watch_sync": settings.plex_watch_sync_interval_min,
|
||||
"plex_watch_reconcile": settings.plex_watch_reconcile_interval_min,
|
||||
"tmdb_enrich": settings.tmdb_enrich_interval_min,
|
||||
"people_enrich": settings.people_enrich_interval_min,
|
||||
"audit_gc": settings.audit_gc_minutes,
|
||||
"log_gc": settings.log_gc_minutes,
|
||||
job: getattr(settings, attr) for job, attr in sysconfig.JOB_INTERVAL_DEFAULTS.items()
|
||||
}
|
||||
|
||||
# Sane bounds for an admin-set interval (minutes).
|
||||
MIN_INTERVAL = 1
|
||||
MAX_INTERVAL = 1440 # one day
|
||||
MIN_INTERVAL = sysconfig.MIN_INTERVAL
|
||||
MAX_INTERVAL = sysconfig.MAX_INTERVAL
|
||||
|
||||
|
||||
def load_intervals(db) -> dict[str, int]:
|
||||
"""Effective per-job intervals: the env/config defaults overlaid with any admin overrides
|
||||
saved in scheduler_settings."""
|
||||
iv = dict(JOB_INTERVALS)
|
||||
for row in db.execute(select(SchedulerSetting)).scalars():
|
||||
if row.job_id in iv and row.interval_minutes:
|
||||
iv[row.job_id] = row.interval_minutes
|
||||
return iv
|
||||
"""Effective per-job intervals: each job's sysconfig override, else its env/config default."""
|
||||
return {job: sysconfig.get_int(db, sysconfig.interval_key(job)) for job in JOB_INTERVALS}
|
||||
|
||||
|
||||
def apply_interval(db, job_id: str, minutes: int) -> int:
|
||||
"""Persist a job's interval override and reschedule the live job immediately (if the
|
||||
scheduler runs in this process). Returns the applied value."""
|
||||
def stage_interval(db, job_id: str, minutes: int) -> int:
|
||||
"""Stage a job's interval override (clamped) in `db`; returns the value. The caller commits it
|
||||
with its audit row, then calls `reschedule` — the live job follows only what was stored."""
|
||||
if job_id not in JOB_INTERVALS:
|
||||
raise KeyError(job_id)
|
||||
minutes = max(MIN_INTERVAL, min(MAX_INTERVAL, int(minutes)))
|
||||
row = db.get(SchedulerSetting, job_id)
|
||||
if row is None:
|
||||
db.add(SchedulerSetting(job_id=job_id, interval_minutes=minutes))
|
||||
else:
|
||||
row.interval_minutes = minutes
|
||||
db.commit()
|
||||
sysconfig.set_value(db, sysconfig.interval_key(job_id), minutes)
|
||||
return minutes
|
||||
|
||||
|
||||
def reschedule(job_id: str, minutes: int) -> None:
|
||||
"""Move the live interval job to `minutes` (if this process registered it)."""
|
||||
if _scheduler is not None and _scheduler.get_job(job_id) is not None:
|
||||
_scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes)
|
||||
log.info("rescheduled job %s to every %s min", job_id, minutes)
|
||||
return minutes
|
||||
|
||||
|
||||
def _record(name: str, **fields) -> None:
|
||||
@@ -112,7 +111,12 @@ def _record(name: str, **fields) -> None:
|
||||
|
||||
|
||||
def _notify_done(
|
||||
db, actor_id: int, job_id: str, status: str, summary: str | None, result: dict | None = None
|
||||
db,
|
||||
actor_id: int,
|
||||
job_id: str,
|
||||
status: RunStatus,
|
||||
summary: str | None,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
"""Post a completion notice to the triggering admin's inbox (manual runs only). Best
|
||||
effort — a notification failure must never mask the job's own outcome."""
|
||||
@@ -130,49 +134,6 @@ def _notify_done(
|
||||
log.exception("failed to write completion notification for job %s", job_id)
|
||||
|
||||
|
||||
def _run_changed(result) -> bool:
|
||||
"""Did an automatic job run actually do something worth an audit row? (No-op polls don't.)
|
||||
A "change" = a truthy NUMERIC count; status-string dicts like {"skipped": "disabled"} (Plex
|
||||
off — the default) are no-ops, not changes, so they don't flood the trail every interval."""
|
||||
if result is None:
|
||||
return False
|
||||
if isinstance(result, dict):
|
||||
# A phased job (backfill's recent/deep, people_enrich's bios/portraits) nests a result per
|
||||
# phase: a count inside a phase is a change too.
|
||||
return any(
|
||||
_run_changed(v) if isinstance(v, dict) else isinstance(v, (int, float)) and bool(v)
|
||||
for v in result.values()
|
||||
)
|
||||
if isinstance(result, (int, float)):
|
||||
return result != 0
|
||||
return bool(result)
|
||||
|
||||
|
||||
def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
||||
"""Write ONE run-summary audit row per job run (never per-item). Manual runs (an admin
|
||||
clicked "run now" → actor_id set) and errors are always logged; automatic runs only when
|
||||
they changed something, so routine no-op polls don't spam the trail. Best-effort."""
|
||||
if actor_id is None and status == "ok" and not _run_changed(result):
|
||||
return
|
||||
# The result as data too: the audit log names it in the reader's language, as the inbox
|
||||
# does; `summary` stays the English line of the row's own summary column.
|
||||
data = _result_data(result)
|
||||
try:
|
||||
audit.record(
|
||||
db,
|
||||
actor_id,
|
||||
AuditAction.JOB_RUN,
|
||||
target_type="job",
|
||||
target_id=job_id,
|
||||
summary=f"{job_id}: {status}" + (f" — {summary}" if summary else ""),
|
||||
after={"status": status, "summary": summary, **({"result": data} if data else {})},
|
||||
)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
log.exception("failed to write audit row for job %s", job_id)
|
||||
|
||||
|
||||
def _job(name: str, fn) -> None:
|
||||
# The one place both run paths converge (manual trigger AND scheduled fire), so claim the run
|
||||
# HERE, atomically: whichever reaches this first wins, the other skips. This is the real
|
||||
@@ -180,71 +141,88 @@ def _job(name: str, fn) -> None:
|
||||
if not _claim_running(name):
|
||||
log.info("job %s already running — skipping this trigger", name)
|
||||
return
|
||||
db = SessionLocal()
|
||||
actor_id = _manual_actor.get() # set only for a manual "run now" trigger
|
||||
_record(
|
||||
name, last_started=datetime.now(timezone.utc).isoformat(), last_error=None, progress=None
|
||||
) # running already claimed
|
||||
trigger = RunTrigger.SCHEDULED if actor_id is None else RunTrigger.MANUAL
|
||||
_record(name, progress=None) # running already claimed
|
||||
db = SessionLocal()
|
||||
run_id = None
|
||||
try:
|
||||
try:
|
||||
run_id = jobruns.start(name, trigger, actor_id)
|
||||
except Exception:
|
||||
# No row to tell it in: the database is what failed. The log is the record.
|
||||
log.exception("job %s not started: its run could not be recorded", name)
|
||||
return
|
||||
_opened(run_id, name)
|
||||
with logs.bind(run_id=run_id):
|
||||
_run(db, name, fn, run_id, actor_id)
|
||||
finally:
|
||||
# ALWAYS release the claim, even if the body raised before its own status update — so a
|
||||
# failure can never wedge the job as permanently "running".
|
||||
if run_id is not None:
|
||||
_closed(run_id)
|
||||
_record(name, running=False, progress=None)
|
||||
db.close()
|
||||
|
||||
|
||||
def _run(db, name: str, fn, run_id: int, actor_id: int | None) -> None:
|
||||
"""The run itself, inside its `job_runs` row: the pause-skip, the work, and its outcome — on
|
||||
the row, and in the triggering admin's inbox for a manual run."""
|
||||
|
||||
def sink(current, total, phase):
|
||||
_record(name, progress={"current": current, "total": total, "phase": phase})
|
||||
|
||||
# Only the work is inside the try: writing its outcome is not the job, and a write that fails
|
||||
# must not turn a run that succeeded into a failed one (see _close).
|
||||
result = None
|
||||
try:
|
||||
if is_sync_paused(db):
|
||||
log.info("job %s skipped (sync paused)", name)
|
||||
_record(
|
||||
name,
|
||||
status="skipped",
|
||||
last_finished=datetime.now(timezone.utc).isoformat(),
|
||||
last_result="paused",
|
||||
result=_PAUSED,
|
||||
progress=None,
|
||||
)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
||||
_audit_run(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
||||
return
|
||||
with progress.bind(sink):
|
||||
result = fn(db)
|
||||
summary = _summarize(result)
|
||||
data = _result_data(result)
|
||||
log.info("job %s -> %s", name, result)
|
||||
_record(
|
||||
name,
|
||||
status="ok",
|
||||
last_finished=datetime.now(timezone.utc).isoformat(),
|
||||
last_result=summary,
|
||||
result=data,
|
||||
progress=None,
|
||||
)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "ok", summary, data)
|
||||
_audit_run(db, actor_id, name, "ok", summary, result)
|
||||
paused = is_sync_paused(db)
|
||||
if not paused:
|
||||
with progress.bind(sink):
|
||||
result = fn(db)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
|
||||
err = str(exc) or exc.__class__.__name__
|
||||
# A failure the job named (progress.JobFailed) rides along as data, as a result does, so the
|
||||
# client says it in the reader's language; `err` stays the English line.
|
||||
failure = (
|
||||
{"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
|
||||
)
|
||||
_record(
|
||||
name,
|
||||
status="error",
|
||||
last_finished=datetime.now(timezone.utc).isoformat(),
|
||||
last_error=err,
|
||||
result=failure,
|
||||
progress=None,
|
||||
)
|
||||
err, failure = _failure(exc)
|
||||
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "error", err, failure)
|
||||
_audit_run(db, actor_id, name, "error", err, failure)
|
||||
finally:
|
||||
# ALWAYS release the claim, even if the body raised before its own status update — so a
|
||||
# failure can never wedge the job as permanently "running".
|
||||
_record(name, running=False)
|
||||
db.close()
|
||||
_notify_done(db, actor_id, name, RunStatus.ERROR, err, failure)
|
||||
return
|
||||
if paused:
|
||||
log.info("job %s skipped (sync paused)", name)
|
||||
_close(run_id, RunStatus.SKIPPED, summary="sync paused", result=_PAUSED)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, RunStatus.SKIPPED, "sync paused", _PAUSED)
|
||||
return
|
||||
summary, data = _finish_ok(name, run_id, result)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, RunStatus.OK, summary, data)
|
||||
|
||||
|
||||
def _close(run_id: int, status: RunStatus, **outcome) -> None:
|
||||
"""Write a run's outcome. Its own failure (the database) is the log's to tell, never the job's:
|
||||
the row then stays `running` until the next start's sweep closes it as interrupted."""
|
||||
try:
|
||||
jobruns.finish(run_id, status, **outcome)
|
||||
except Exception:
|
||||
log.exception("run %s: its outcome (%s) could not be recorded", run_id, status)
|
||||
|
||||
|
||||
def _finish_ok(name: str, run_id: int, result) -> tuple[str | None, dict | None]:
|
||||
"""Close a run that returned: its result on the row, as data and as the English one-liner."""
|
||||
summary = _summarize(result)
|
||||
data = _result_data(result)
|
||||
log.info("job %s -> %s", name, result)
|
||||
_close(run_id, RunStatus.OK, summary=summary, result=data)
|
||||
return summary, data
|
||||
|
||||
|
||||
def _failure(exc: Exception) -> tuple[str, dict | None]:
|
||||
"""A raised run's English error line, and the failure the job named (progress.JobFailed) as
|
||||
data, as a result is, so the client says it in the reader's language."""
|
||||
err = str(exc) or exc.__class__.__name__
|
||||
failure = {"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
|
||||
return err, failure
|
||||
|
||||
|
||||
# A skipped run's result, in the same shape as a job's own `{"skipped": …}`.
|
||||
@@ -271,11 +249,13 @@ def _summarize(result) -> str | None:
|
||||
return str(result)
|
||||
|
||||
|
||||
def scheduler_snapshot() -> dict:
|
||||
"""What the scheduler is doing, for the admin dashboard. `running_here` is False in
|
||||
environments where the scheduler is disabled (e.g. local dev shares the prod-ish DB but
|
||||
runs no scheduler), so the UI can say so while still showing DB-derived queue/quota."""
|
||||
running_here = _scheduler is not None
|
||||
def scheduler_snapshot(db) -> dict:
|
||||
"""What the scheduler is doing, for the admin dashboard: each job's live state from this process
|
||||
and its last runs from `job_runs`, so a restart no longer blanks "when did it last run, and how
|
||||
did it go"; and the same for the ad-hoc runs users start on one target. `running_here` is False where the scheduler runs no interval jobs
|
||||
(SCHEDULER_ENABLED=false), so the UI can say so while still showing the DB-derived history,
|
||||
queue and quota."""
|
||||
running_here = _scheduler is not None and settings.scheduler_enabled
|
||||
next_runs: dict[str, str | None] = {}
|
||||
live_intervals: dict[str, int] = {}
|
||||
if _scheduler is not None:
|
||||
@@ -287,6 +267,12 @@ def scheduler_snapshot() -> dict:
|
||||
live_intervals[job.id] = round(iv.total_seconds() / 60)
|
||||
with _activity_lock:
|
||||
acts = {k: dict(v) for k, v in _activity.items()}
|
||||
# Open before the read or after it: a run that started or ended while the rows were read is
|
||||
# still counted as the open run it was, never as a stray one.
|
||||
open_runs = _open_run_ids()
|
||||
recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS)
|
||||
open_runs |= _open_run_ids()
|
||||
going_id = max((r for r, j in open_runs.items() if j == CHANNEL_BACKFILL), default=None)
|
||||
jobs = []
|
||||
for job_id, default_interval in JOB_INTERVALS.items():
|
||||
a = acts.get(job_id, {})
|
||||
@@ -296,16 +282,94 @@ def scheduler_snapshot() -> dict:
|
||||
"interval_minutes": live_intervals.get(job_id, default_interval),
|
||||
"next_run": next_runs.get(job_id),
|
||||
"running": a.get("running", False),
|
||||
"status": a.get("status"),
|
||||
"last_started": a.get("last_started"),
|
||||
"last_finished": a.get("last_finished"),
|
||||
"last_result": a.get("last_result"),
|
||||
"result": a.get("result"),
|
||||
"last_error": a.get("last_error"),
|
||||
"progress": a.get("progress"),
|
||||
**_last_runs(recent.get(job_id, []), open_runs),
|
||||
}
|
||||
)
|
||||
return {"running_here": running_here, "enabled": settings.scheduler_enabled, "jobs": jobs}
|
||||
# Work a user starts on one target: no interval, no "run now" (it needs its target), but its
|
||||
# runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the
|
||||
# target's own (sync/runner), so "running" is the runs this process has open: two channels can
|
||||
# backfill at once, so a run still going counts even when a newer one already ended. That run then
|
||||
# leads the row, so its channel, its "no outcome yet" and its dot are one run's, never the
|
||||
# newer run's outcome under the older one's channel.
|
||||
backfills = recent.get(CHANNEL_BACKFILL, [])
|
||||
going = db.get(JobRun, going_id) if going_id is not None else None
|
||||
if going is not None and going.status != RunStatus.RUNNING:
|
||||
going = None # its outcome is written; only the bookkeeping has not caught up
|
||||
if going is not None:
|
||||
backfills = [going, *(r for r in backfills if r.id != going.id)][:RECENT_RUNS]
|
||||
shown = backfills[0] if backfills else None
|
||||
ad_hoc = [
|
||||
{
|
||||
"id": CHANNEL_BACKFILL,
|
||||
"running": going is not None,
|
||||
"target": shown.target if shown else None,
|
||||
"target_title": (
|
||||
db.scalar(select(Channel.title).where(Channel.id == shown.target))
|
||||
if shown is not None and shown.target
|
||||
else None
|
||||
),
|
||||
**_last_runs(backfills, open_runs),
|
||||
}
|
||||
]
|
||||
return {
|
||||
"running_here": running_here,
|
||||
"enabled": settings.scheduler_enabled,
|
||||
"jobs": jobs,
|
||||
"ad_hoc": ad_hoc,
|
||||
}
|
||||
|
||||
|
||||
# How many past runs a Scheduler row shows as dots; the full history is the run list.
|
||||
RECENT_RUNS = 3
|
||||
|
||||
|
||||
def _seen(run, open_runs: dict[int, str]) -> RunStatus:
|
||||
"""A run's status as the page tells it. A row `running` that no run here has open is one whose
|
||||
outcome failed to write: the next start's sweep closes it as interrupted, so it reads so now —
|
||||
not as a run still going beside a row that says nothing runs."""
|
||||
if run.status == RunStatus.RUNNING and run.id not in open_runs:
|
||||
return RunStatus.INTERRUPTED
|
||||
return RunStatus(run.status)
|
||||
|
||||
|
||||
def _last_runs(runs: list, open_runs: dict[int, str]) -> dict:
|
||||
"""A row's run fields from its newest runs (newest first): how the last one went, and the few
|
||||
before it for the history dots."""
|
||||
run = runs[0] if runs else None
|
||||
status = _seen(run, open_runs) if run is not None else None
|
||||
# A run still going has no outcome yet: only an ended run says how it went.
|
||||
ended = run if status is not None and status != RunStatus.RUNNING else None
|
||||
return {
|
||||
"status": status if ended else None,
|
||||
"last_started": iso(run.started_at) if run else None,
|
||||
# A stray row has no end of its own: it ended, as far as anyone can tell, where it began.
|
||||
"last_finished": iso(ended.finished_at or ended.started_at) if ended else None,
|
||||
"last_result": ended.summary if ended else None,
|
||||
# The outcome as data: the result, or on a failed run the failure the job named.
|
||||
"result": _outcome_data(ended),
|
||||
"last_error": ended.error if ended else None,
|
||||
"last_trigger": run.trigger if run else None,
|
||||
"last_run_id": run.id if run else None,
|
||||
"quota_units": ended.quota_units if ended else None,
|
||||
"recent": [_run_brief(r, open_runs) for r in runs],
|
||||
}
|
||||
|
||||
|
||||
def _run_brief(run, open_runs: dict[int, str]) -> dict:
|
||||
return {
|
||||
"id": run.id,
|
||||
"status": _seen(run, open_runs),
|
||||
"trigger": run.trigger,
|
||||
"started_at": iso(run.started_at),
|
||||
"finished_at": iso(run.finished_at),
|
||||
}
|
||||
|
||||
|
||||
def _outcome_data(run) -> dict | None:
|
||||
if run is None:
|
||||
return None
|
||||
return run.failure if run.status == RunStatus.ERROR else run.result
|
||||
|
||||
|
||||
def _rss_job() -> None:
|
||||
@@ -408,15 +472,16 @@ def _people_enrich_job() -> None:
|
||||
|
||||
|
||||
def _audit_gc_job() -> None:
|
||||
# The history cleanup: audit-log rows past audit_retention_days and YouTube quota history past
|
||||
# quota_retention_days (0 = keep forever, each). Pure DB, no quota.
|
||||
# The history cleanup: audit-log rows past audit_retention_days, YouTube quota history past
|
||||
# quota_retention_days and finished runs past job_run_retention_days (0 = keep forever, each).
|
||||
# Pure DB, no quota.
|
||||
def work(db):
|
||||
from app import sysconfig
|
||||
|
||||
reaped = audit.gc(db, sysconfig.get_int(db, "audit_retention_days"))
|
||||
quota_reaped = quota.gc(db, sysconfig.get_int(db, "quota_retention_days"))
|
||||
runs_reaped = jobruns.gc(db, sysconfig.get_int(db, "job_run_retention_days"))
|
||||
db.commit()
|
||||
return {"reaped": reaped, "quota_reaped": quota_reaped}
|
||||
return {"reaped": reaped, "quota_reaped": quota_reaped, "runs_reaped": runs_reaped}
|
||||
|
||||
_job("audit_gc", work)
|
||||
|
||||
@@ -425,7 +490,6 @@ def _log_gc_job() -> None:
|
||||
# Prune log_events rows older than log_retention_days — the limit the privacy policy states, so
|
||||
# this runs hourly rather than daily. Pure DB, no quota.
|
||||
def work(db):
|
||||
from app import sysconfig
|
||||
|
||||
days = sysconfig.get_int(db, "log_retention_days")
|
||||
reaped = logsink.gc(db, days)
|
||||
@@ -485,15 +549,29 @@ def running_job_ids() -> set[str]:
|
||||
return {jid for jid, a in _activity.items() if a.get("running")}
|
||||
|
||||
|
||||
def trigger_job(job_id: str, actor_id: int | None = None) -> str:
|
||||
"""Run a job immediately in a background thread, independent of its interval schedule.
|
||||
The wrapper handles its own DB session, activity tracking and pause-skip, so the live
|
||||
dashboard reflects the run. Returns "started", "already_running", or raises KeyError for
|
||||
an unknown job. Refusing a concurrent run keeps a manual trigger from overlapping a
|
||||
scheduled run (APScheduler enforces max_instances=1 for the scheduled side).
|
||||
def _as_actor(fn: Callable[[], None], actor_id: int | None) -> Callable[[], None]:
|
||||
"""`fn` run as `actor_id`'s manual run. The actor is RESET afterwards: the executor's pool
|
||||
threads are reused and keep their context from task to task, so an actor left set would make
|
||||
the next scheduled run that thread picks up look like this admin's manual run."""
|
||||
|
||||
`actor_id` (the admin who clicked "run now") is stashed in a contextvar inside the new
|
||||
thread so the run posts a completion notification to that admin's inbox."""
|
||||
def run() -> None:
|
||||
token = _manual_actor.set(actor_id)
|
||||
try:
|
||||
fn()
|
||||
finally:
|
||||
_manual_actor.reset(token)
|
||||
|
||||
return run
|
||||
|
||||
|
||||
def trigger_job(job_id: str, actor_id: int | None = None) -> str:
|
||||
"""Run a job now, independent of its interval schedule: a one-shot job on the same scheduler,
|
||||
so a manual run shares the scheduled runs' executor and shutdown — no thread of its own. The
|
||||
wrapper handles its own DB session, run row and pause-skip. Returns "started",
|
||||
"already_running", or raises KeyError for an unknown job.
|
||||
|
||||
`actor_id` (the admin who clicked "run now") is set in a contextvar inside the run's thread, so
|
||||
the run is recorded as theirs and posts a completion notification to their inbox."""
|
||||
fn = JOB_FUNCS.get(job_id)
|
||||
if fn is None:
|
||||
raise KeyError(job_id)
|
||||
@@ -502,13 +580,18 @@ def trigger_job(job_id: str, actor_id: int | None = None) -> str:
|
||||
# _job claims, sees it's taken, and skips — no double run.
|
||||
if _is_running(job_id):
|
||||
return "already_running"
|
||||
sched = _scheduler
|
||||
if sched is None:
|
||||
# Only between shutdown and exit: start_scheduler runs at every API startup.
|
||||
raise RuntimeError("the scheduler is not running")
|
||||
|
||||
def run() -> None:
|
||||
if actor_id is not None:
|
||||
_manual_actor.set(actor_id)
|
||||
fn()
|
||||
|
||||
threading.Thread(target=run, name=f"trigger-{job_id}", daemon=True).start()
|
||||
try:
|
||||
# One id per job: a second click before the first one-shot has even started is the same
|
||||
# run, not a queued second one. misfire_grace_time=None: a busy pool delays it, never drops it.
|
||||
sched.add_job(_as_actor(fn, actor_id), "date", id=f"{job_id}@now", misfire_grace_time=None)
|
||||
# silent-ok: not a failure — the answer is the 409 the admin sees ("already running").
|
||||
except ConflictingIdError:
|
||||
return "already_running"
|
||||
return "started"
|
||||
|
||||
|
||||
@@ -521,22 +604,95 @@ def trigger_all(actor_id: int | None = None) -> list[str]:
|
||||
return started
|
||||
|
||||
|
||||
def start_scheduler() -> None:
|
||||
global _scheduler
|
||||
if not settings.scheduler_enabled or _scheduler is not None:
|
||||
return
|
||||
# Effective intervals = env defaults overlaid with any admin overrides from the DB.
|
||||
db = SessionLocal()
|
||||
# Work a user starts on one target outside the job list — today the "get full history" backfill of
|
||||
# one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and
|
||||
# spent is in `job_runs` like any job's, not a background task only the log ever heard of.
|
||||
CHANNEL_BACKFILL = "channel_backfill"
|
||||
# Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple).
|
||||
AD_HOC_JOBS = (CHANNEL_BACKFILL,)
|
||||
|
||||
|
||||
def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None:
|
||||
"""Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its
|
||||
own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so
|
||||
a second request before the first has even started is that same run."""
|
||||
if job not in AD_HOC_JOBS:
|
||||
raise KeyError(job) # an unnamed job would show on the page as its bare id
|
||||
sched = _scheduler
|
||||
if sched is None:
|
||||
# Only between shutdown and exit: start_scheduler runs at every API startup.
|
||||
raise RuntimeError("the scheduler is not running")
|
||||
try:
|
||||
iv = load_intervals(db)
|
||||
sched.add_job(
|
||||
_ad_hoc,
|
||||
"date",
|
||||
args=(job, fn, actor_id, target),
|
||||
id=f"{job}@{target}",
|
||||
misfire_grace_time=None,
|
||||
)
|
||||
# silent-ok: not a failure — the run for this target is already queued, and it does this work.
|
||||
except ConflictingIdError:
|
||||
log.info("ad-hoc %s for %s already queued", job, target)
|
||||
|
||||
|
||||
def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> None:
|
||||
try:
|
||||
run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target)
|
||||
except Exception:
|
||||
# No row to tell it in: the database is what failed. The log is the record.
|
||||
log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target)
|
||||
return
|
||||
_opened(run_id, job)
|
||||
try:
|
||||
_ad_hoc_work(job, fn, run_id)
|
||||
finally:
|
||||
db.close()
|
||||
scheduler = BackgroundScheduler(timezone="UTC")
|
||||
for job_id, fn in JOB_FUNCS.items():
|
||||
scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id)
|
||||
_closed(run_id)
|
||||
|
||||
|
||||
def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None:
|
||||
with logs.bind(run_id=run_id):
|
||||
# As in _run: only the work is inside the try.
|
||||
try:
|
||||
result = fn()
|
||||
except Exception as exc:
|
||||
log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
|
||||
err, failure = _failure(exc)
|
||||
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
|
||||
return
|
||||
_finish_ok(job, run_id, result)
|
||||
|
||||
|
||||
# Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of
|
||||
# 10 would queue a scheduled fire behind long manual runs.
|
||||
_POOL_SIZE = len(JOB_FUNCS) + 8
|
||||
|
||||
|
||||
def start_scheduler() -> None:
|
||||
"""Start this process's scheduler. It always starts, so "run now" works on every install; only
|
||||
with SCHEDULER_ENABLED does it also register the recurring interval jobs. Before anything can
|
||||
run, every run a previous process left open is closed as interrupted — one API process serves a
|
||||
database (app/jobruns.py), so an open run here is a dead process's."""
|
||||
global _scheduler
|
||||
if _scheduler is not None:
|
||||
return
|
||||
left = jobruns.interrupt_leftovers()
|
||||
if left:
|
||||
log.warning("%s run(s) left running by the previous process marked interrupted", left)
|
||||
scheduler = BackgroundScheduler(
|
||||
timezone="UTC", executors={"default": ThreadPoolExecutor(_POOL_SIZE)}
|
||||
)
|
||||
if settings.scheduler_enabled:
|
||||
# Effective intervals = env defaults overlaid with any admin overrides from the DB.
|
||||
db = SessionLocal()
|
||||
try:
|
||||
iv = load_intervals(db)
|
||||
finally:
|
||||
db.close()
|
||||
for job_id, fn in JOB_FUNCS.items():
|
||||
scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id)
|
||||
scheduler.start()
|
||||
_scheduler = scheduler
|
||||
log.info("scheduler started")
|
||||
log.info("scheduler started (interval jobs %s)", "on" if settings.scheduler_enabled else "off")
|
||||
|
||||
|
||||
def shutdown_scheduler() -> None:
|
||||
|
||||
+14
-87
@@ -8,11 +8,13 @@ regenerated freely; user tags are never touched here.
|
||||
|
||||
import re
|
||||
|
||||
from sqlalchemy import and_, exists, func, select
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, progress, sysconfig
|
||||
from app.models import Channel, ChannelTag, SavedView, Tag, Video
|
||||
from app.models import Channel, Video
|
||||
from app.reader import queries as reader_queries
|
||||
from app.reader import service as reader_service
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
@@ -209,71 +211,30 @@ def compute_channel_topics(db: Session, channel: Channel) -> set[str]:
|
||||
return topics
|
||||
|
||||
|
||||
def ensure_system_tag(db: Session, name: str, category: str, color: str | None) -> Tag:
|
||||
tag = db.execute(
|
||||
select(Tag).where(Tag.user_id.is_(None), Tag.name == name)
|
||||
).scalar_one_or_none()
|
||||
if tag is None:
|
||||
tag = Tag(user_id=None, name=name, category=category, color=color)
|
||||
db.add(tag)
|
||||
db.flush()
|
||||
return tag
|
||||
|
||||
|
||||
def apply_channel_autotags(db: Session, channel: Channel) -> None:
|
||||
desired: dict[int, float | None] = {}
|
||||
|
||||
lang, confidence = detect_channel_language(db, channel)
|
||||
if lang:
|
||||
tag = ensure_system_tag(db, LANG_NAMES.get(lang, lang.upper()), "language", LANG_COLOR)
|
||||
tag = reader_service.ensure_system_tag(
|
||||
db, LANG_NAMES.get(lang, lang.upper()), "language", LANG_COLOR
|
||||
)
|
||||
desired[tag.id] = confidence
|
||||
|
||||
for topic in compute_channel_topics(db, channel):
|
||||
tag = ensure_system_tag(db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR))
|
||||
tag = reader_service.ensure_system_tag(
|
||||
db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR)
|
||||
)
|
||||
desired.setdefault(tag.id, None)
|
||||
|
||||
existing = {
|
||||
ct.tag_id: ct
|
||||
for ct in db.execute(
|
||||
select(ChannelTag).where(
|
||||
ChannelTag.channel_id == channel.id,
|
||||
ChannelTag.user_id.is_(None),
|
||||
ChannelTag.source == "auto",
|
||||
)
|
||||
).scalars()
|
||||
}
|
||||
for tag_id, ct in existing.items():
|
||||
if tag_id not in desired:
|
||||
db.delete(ct)
|
||||
for tag_id, conf in desired.items():
|
||||
ct = existing.get(tag_id)
|
||||
if ct is None:
|
||||
db.add(
|
||||
ChannelTag(
|
||||
channel_id=channel.id,
|
||||
tag_id=tag_id,
|
||||
user_id=None,
|
||||
source="auto",
|
||||
confidence=conf,
|
||||
)
|
||||
)
|
||||
else:
|
||||
ct.confidence = conf
|
||||
reader_service.set_auto_tags(db, channel.id, desired)
|
||||
db.commit()
|
||||
|
||||
|
||||
def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
|
||||
query = select(Channel)
|
||||
if only_missing:
|
||||
query = query.where(
|
||||
~exists().where(
|
||||
and_(
|
||||
ChannelTag.channel_id == Channel.id,
|
||||
ChannelTag.source == "auto",
|
||||
ChannelTag.user_id.is_(None),
|
||||
)
|
||||
)
|
||||
)
|
||||
query = query.where(~reader_queries.has_auto_tags(Channel.id))
|
||||
channels = db.execute(query).scalars().all()
|
||||
tagged = 0
|
||||
for i, channel in enumerate(channels, 1):
|
||||
@@ -284,40 +245,6 @@ def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
|
||||
db.rollback()
|
||||
log.exception("Auto-tagging failed for channel %s", channel.id)
|
||||
progress.report(i, len(channels), "autotag")
|
||||
removed = _cleanup_orphan_system_tags(db)
|
||||
return {"channels_tagged": tagged, "orphan_tags_removed": removed}
|
||||
|
||||
|
||||
def _view_tag_ids(filters: object) -> list[int]:
|
||||
"""The tag ids a stored view's filters name; anything else in that slot names none."""
|
||||
tags = filters.get("tags") if isinstance(filters, dict) else None
|
||||
if not isinstance(tags, list):
|
||||
return []
|
||||
return [t for t in tags if type(t) is int] # `type`, not isinstance: True is not a tag id
|
||||
|
||||
|
||||
def _cleanup_orphan_system_tags(db: Session) -> int:
|
||||
"""Delete system tags that no longer link to any channel (e.g. after a mapping change) — but not
|
||||
one a saved feed view filters on. Views hold tag IDS, and a deleted tag comes back under a new id
|
||||
when a channel earns it again, so the view's filter would point at nothing; the feed drops an
|
||||
unknown id, and the view showed the whole feed under its old name.
|
||||
|
||||
The views are read defensively, and the kept ids are filtered here rather than in SQL: a view
|
||||
stored before its route checked the shape (or written around it) must not raise for everyone's
|
||||
job, nor push a `NOT IN` past the driver's bind-parameter limit."""
|
||||
in_views = {
|
||||
t
|
||||
for (filters,) in db.execute(select(SavedView.filters).where(SavedView.source == "feed"))
|
||||
for t in _view_tag_ids(filters)
|
||||
}
|
||||
orphans = [
|
||||
tag
|
||||
for tag in db.execute(
|
||||
select(Tag).where(Tag.user_id.is_(None), ~exists().where(ChannelTag.tag_id == Tag.id))
|
||||
).scalars()
|
||||
if tag.id not in in_views
|
||||
]
|
||||
for tag in orphans:
|
||||
db.delete(tag)
|
||||
removed = reader_service.prune_orphan_system_tags(db)
|
||||
db.commit()
|
||||
return len(orphans)
|
||||
return {"channels_tagged": tagged, "orphan_tags_removed": removed}
|
||||
|
||||
@@ -14,15 +14,8 @@ from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.models import (
|
||||
Channel,
|
||||
ExploredChannel,
|
||||
PlaylistItem,
|
||||
SearchFind,
|
||||
Subscription,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.models import Channel, ExploredChannel, SearchFind, Subscription, Video
|
||||
from app.reader import queries as reader_queries
|
||||
from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
|
||||
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
|
||||
|
||||
@@ -129,15 +122,12 @@ def purge_unkept_explored(db: Session) -> dict:
|
||||
if not orphan_ids:
|
||||
return {"expired": expired, "videos_deleted": 0, "channels_deleted": 0}
|
||||
|
||||
state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
|
||||
pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
|
||||
find_exists = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists()
|
||||
videos_deleted = db.execute(
|
||||
delete(Video).where(
|
||||
Video.channel_id.in_(orphan_ids),
|
||||
Video.via_explore.is_(True),
|
||||
~state_exists,
|
||||
~pl_exists,
|
||||
~reader_queries.kept_by_a_reader(Video.id),
|
||||
~find_exists,
|
||||
)
|
||||
).rowcount
|
||||
@@ -166,15 +156,12 @@ def purge_unkept_search(db: Session, grace_days: int | None = None) -> dict:
|
||||
0 quota."""
|
||||
grace = sysconfig.get_int(db, "search_grace_days") if grace_days is None else grace_days
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=grace)
|
||||
state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
|
||||
pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
|
||||
sub_exists = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
|
||||
deleted = db.execute(
|
||||
delete(Video).where(
|
||||
Video.via_search.is_(True),
|
||||
Video.created_at < cutoff,
|
||||
~state_exists,
|
||||
~pl_exists,
|
||||
~reader_queries.kept_by_a_reader(Video.id),
|
||||
~sub_exists,
|
||||
)
|
||||
).rowcount
|
||||
|
||||
@@ -27,7 +27,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, progress, quota, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import Playlist, PlaylistItem, Video
|
||||
from app.models import Video
|
||||
from app.reader import queries as reader_queries
|
||||
from app.notifications import create_notification
|
||||
from app.sync.runner import get_service_user
|
||||
from app.sync.videos import apply_video_details, parse_dt
|
||||
@@ -88,11 +89,7 @@ def _collect_user_impact(db: Session, videos: list[Video]) -> dict[int, list[dic
|
||||
# Anyone who had it in a playlist — which now also covers "saved": Watch later became a
|
||||
# built-in playlist, so "saved" is no longer a VideoState status (the old status=="saved"
|
||||
# scan matched nothing and under-counted impacted users).
|
||||
for user_id, video_id in db.execute(
|
||||
select(Playlist.user_id, PlaylistItem.video_id)
|
||||
.join(PlaylistItem, PlaylistItem.playlist_id == Playlist.id)
|
||||
.where(PlaylistItem.video_id.in_(ids))
|
||||
):
|
||||
for user_id, video_id in reader_queries.playlisted_by(db, ids):
|
||||
impact.setdefault(user_id, set()).add(video_id)
|
||||
return {
|
||||
user_id: [
|
||||
|
||||
@@ -12,7 +12,6 @@ import json
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import TypeVar
|
||||
|
||||
from sqlalchemy import Engine, create_engine, delete, func, select
|
||||
@@ -21,12 +20,14 @@ from sqlalchemy.pool import NullPool
|
||||
|
||||
from app import logs, progress, quota
|
||||
from app.auth import has_read_scope
|
||||
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
||||
from app.models import User, Video
|
||||
from app.reader.models import Playlist, PlaylistItem
|
||||
from app.sync.playlist_merge import merge_plan
|
||||
from app.sync.runner import token_users
|
||||
from app.sync.subscriptions import blank_listing, refuse_blank_listing, vanished
|
||||
from app.sync.videos import apply_video_details, parse_dt
|
||||
from app.sync.videos import ensure_videos
|
||||
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
|
||||
from app.userscope import owned_by
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
||||
@@ -227,55 +228,11 @@ def recompute_dirty(db: Session, pl: Playlist) -> None:
|
||||
pl.dirty = pl.synced_fingerprint != current_fingerprint(db, pl)
|
||||
|
||||
|
||||
def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
|
||||
"""Make sure every given video id exists in the catalog, fetching + ingesting the
|
||||
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
|
||||
the API won't return are simply left out."""
|
||||
if not video_ids:
|
||||
return
|
||||
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
|
||||
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
|
||||
if not missing:
|
||||
return
|
||||
items = yt.get_videos(missing)
|
||||
chan_ids = {
|
||||
it.get("snippet", {}).get("channelId")
|
||||
for it in items
|
||||
if it.get("snippet", {}).get("channelId")
|
||||
}
|
||||
existing_ch = set(
|
||||
db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
|
||||
)
|
||||
now = datetime.now(timezone.utc)
|
||||
for it in items:
|
||||
sn = it.get("snippet", {})
|
||||
cid = sn.get("channelId")
|
||||
if cid and cid not in existing_ch:
|
||||
db.add(Channel(id=cid, title=sn.get("channelTitle")))
|
||||
existing_ch.add(cid)
|
||||
db.flush()
|
||||
seen: set[str] = set()
|
||||
for it in items:
|
||||
vid = it.get("id")
|
||||
sn = it.get("snippet", {})
|
||||
cid = sn.get("channelId")
|
||||
if not vid or not cid or vid in seen:
|
||||
continue
|
||||
seen.add(vid)
|
||||
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
|
||||
# fallback here too so the constructor never carries an explicit NULL.
|
||||
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
|
||||
apply_video_details(v, it)
|
||||
v.enriched_at = now
|
||||
db.add(v)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _replace_items_from_youtube(db: Session, pl: Playlist, ids: list[str]) -> list[str]:
|
||||
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
|
||||
replace): keep only those present in the catalog in their YT order (de-duplicated), swap the
|
||||
items out, and reset the synced fingerprint + dirty flag. The caller has ingested `ids`
|
||||
(`_ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
|
||||
(`sync.videos.ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
|
||||
after: an ingest in here re-fetched every id YouTube will not return (a private or deleted
|
||||
video) and its commit let go of the lock before the replace. Returns the final ordered ids."""
|
||||
present = (
|
||||
@@ -346,7 +303,9 @@ def pull_user_playlists(db: Session, user: User) -> dict:
|
||||
existing = {
|
||||
pl.yt_playlist_id: pl
|
||||
for pl in db.execute(
|
||||
select(Playlist).where(Playlist.user_id == user.id, Playlist.source == "youtube")
|
||||
select(Playlist).where(
|
||||
owned_by(Playlist.user_id, user.id), Playlist.source == "youtube"
|
||||
)
|
||||
).scalars()
|
||||
}
|
||||
# YouTube playlists that we already own as local (exported) playlists must not be
|
||||
@@ -354,7 +313,7 @@ def pull_user_playlists(db: Session, user: User) -> dict:
|
||||
owned_yt_ids = set(
|
||||
db.execute(
|
||||
select(Playlist.yt_playlist_id).where(
|
||||
Playlist.user_id == user.id,
|
||||
owned_by(Playlist.user_id, user.id),
|
||||
Playlist.source == "local",
|
||||
Playlist.yt_playlist_id.is_not(None),
|
||||
)
|
||||
@@ -401,7 +360,7 @@ def pull_user_playlists(db: Session, user: User) -> dict:
|
||||
continue
|
||||
# Ingest first — it commits — so the row lock below is held from the dirty check
|
||||
# to the commit that replaces the items.
|
||||
_ensure_videos(db, yt, ids)
|
||||
ensure_videos(db, yt, ids)
|
||||
# `dirty` read above is a snapshot from before the YouTube calls (sessions keep
|
||||
# loaded rows past a commit): a remove or reorder made meanwhile must not be
|
||||
# replaced by YouTube's order. Item writes lock the playlist row, so this waits for
|
||||
@@ -612,7 +571,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
prev_unavailable = parse_unavailable(pl)
|
||||
still_listed = [v for v in yt_ids if v in prev_unavailable]
|
||||
if still_listed:
|
||||
_ensure_videos(db, yt, still_listed)
|
||||
ensure_videos(db, yt, still_listed)
|
||||
healed = set(
|
||||
db.execute(select(Video.id).where(Video.id.in_(still_listed))).scalars().all()
|
||||
)
|
||||
@@ -624,7 +583,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
# --- Siftlode side (free, direct): pull YouTube-adds in, drop YouTube-deletes, land the
|
||||
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
||||
# YouTube write trips the breaker.
|
||||
_ensure_videos(db, yt, mp.add_to_local)
|
||||
ensure_videos(db, yt, mp.add_to_local)
|
||||
# `local` was read before the YouTube calls above: an add or remove committed meanwhile
|
||||
# (another tab) would be undone by the replace below. Lock the list — the ingest's commit is
|
||||
# behind us — and plan again from what is there now. Nothing new needs ingesting: a video
|
||||
@@ -749,7 +708,7 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
||||
)
|
||||
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
||||
refuse_blank_listing(ids, snippet.get("item_count"))
|
||||
_ensure_videos(db, yt, ids)
|
||||
ensure_videos(db, yt, ids)
|
||||
_lock_or_gone(db, pl)
|
||||
pl.name = snippet.get("title") or pl.name
|
||||
ordered = _replace_items_from_youtube(db, pl, ids)
|
||||
|
||||
+22
-21
@@ -437,7 +437,7 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
|
||||
|
||||
# Channels being deep-backfilled RIGHT NOW by EITHER path — the interactive backfill_channel_now (a
|
||||
# FastAPI BackgroundTasks threadpool thread) OR the scheduler's run_deep_backfill (its own thread).
|
||||
# one-shot run on the scheduler's pool, see scheduler.run_ad_hoc) OR the scheduler's run_deep_backfill.
|
||||
# BOTH register the channel they're paging (under the lock) so the other skips it: no two runs page
|
||||
# the same channel's `backfill_cursor` concurrently, whichever started first. `_immediate_running`
|
||||
# counts only the interactive path, capped so one user's burst can't drain the shared budget (the
|
||||
@@ -448,9 +448,9 @@ _backfill_lock = threading.Lock()
|
||||
_MAX_CONCURRENT_IMMEDIATE = 2
|
||||
|
||||
|
||||
def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> None:
|
||||
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
|
||||
background task with its OWN session, so the full catalogue lands in seconds instead of over the
|
||||
def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> dict:
|
||||
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, as its
|
||||
own recorded run with its OWN session, so the full catalogue lands in seconds instead of over the
|
||||
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). max_pages is high enough to finish
|
||||
any realistic channel in a single run (400 pages = 20 000 videos; e.g. BBC Earth's ~3400 = 69
|
||||
pages — the old 60-page cap stopped one page short of large channels, so their tail dribbled in
|
||||
@@ -459,34 +459,35 @@ def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) ->
|
||||
Quota-guarded, capped to _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the
|
||||
shared backfill budget by opting many channels in at once (the overflow, and any channel too large
|
||||
to finish under the reserve, is finished by the scheduler since deep_requested stays set), and
|
||||
mutually excluded per-channel with the scheduler via _backfill_in_progress. Best-effort — logs and
|
||||
swallows failures."""
|
||||
mutually excluded per-channel with the scheduler via _backfill_in_progress.
|
||||
|
||||
Returns what the run did, as a job result: the counts, or `{"skipped": <why>}` when it left the
|
||||
channel to the scheduler. A failure raises — the run that wraps this records it."""
|
||||
global _immediate_running
|
||||
with _backfill_lock:
|
||||
if channel_id in _backfill_in_progress or _immediate_running >= _MAX_CONCURRENT_IMMEDIATE:
|
||||
return # already backfilling this channel, or too many interactive runs — leave it to the scheduler
|
||||
if channel_id in _backfill_in_progress:
|
||||
return {"skipped": "channel busy"}
|
||||
if _immediate_running >= _MAX_CONCURRENT_IMMEDIATE:
|
||||
return {"skipped": "at capacity"}
|
||||
_backfill_in_progress.add(channel_id)
|
||||
_immediate_running += 1
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
user = get_service_user(db)
|
||||
if user is None:
|
||||
return
|
||||
return {"skipped": "no service user"}
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
|
||||
return
|
||||
return {"skipped": "nothing to do"}
|
||||
if quota.backfill_reserve_reached():
|
||||
return
|
||||
try:
|
||||
# Attribute the spend to the requesting user under the full-backfill action, so it
|
||||
# shows up correctly in the usage breakdown (not the default OTHER/system bucket).
|
||||
with quota.attribute(user_id, quota.QuotaAction.VIDEOS_BACKFILL_FULL):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
backfill_channel_deep(db, yt, channel, max_pages)
|
||||
reconcile_full_history(db)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
log.exception("Immediate backfill failed for channel %s", channel_id)
|
||||
return {"skipped": "low quota"}
|
||||
# Attribute the spend to the requesting user under the full-backfill action, so it
|
||||
# shows up correctly in the usage breakdown (not the default OTHER/system bucket).
|
||||
with quota.attribute(user_id, quota.QuotaAction.VIDEOS_BACKFILL_FULL):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
added = backfill_channel_deep(db, yt, channel, max_pages)
|
||||
reconciled = reconcile_full_history(db)
|
||||
return {"videos_added": added, "reconciled_full": reconciled}
|
||||
finally:
|
||||
with _backfill_lock:
|
||||
_backfill_in_progress.discard(channel_id)
|
||||
|
||||
@@ -23,7 +23,9 @@ from collections import Counter
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import ChannelTag, Subscription, Tag, User
|
||||
from app.models import Subscription, User
|
||||
from app.reader.models import ChannelTag, Tag
|
||||
from app.userscope import owned_by
|
||||
|
||||
# A user tag needs at least this many manually-tagged example channels before we trust its
|
||||
# profile — fewer is noise (one gaming-cooking channel shouldn't make "cooking" mean "gaming").
|
||||
@@ -57,7 +59,9 @@ def compute_tag_suggestions(db: Session, user: User) -> dict[str, list[dict]]:
|
||||
# Channel Manager shows ghost chips only on non-hidden rows, so suggesting for hidden channels would
|
||||
# inflate the header count past what's shown and let bulk-accept tag channels the user can't see).
|
||||
subs = db.execute(
|
||||
select(Subscription.channel_id, Subscription.hidden).where(Subscription.user_id == user.id)
|
||||
select(Subscription.channel_id, Subscription.hidden).where(
|
||||
owned_by(Subscription.user_id, user.id)
|
||||
)
|
||||
).all()
|
||||
subscribed: set[str] = {cid for cid, _ in subs}
|
||||
hidden: set[str] = {cid for cid, h in subs if h}
|
||||
@@ -74,7 +78,7 @@ def compute_tag_suggestions(db: Session, user: User) -> dict[str, list[dict]]:
|
||||
user_tagged: set[str] = set()
|
||||
for tag_id, cid in db.execute(
|
||||
select(ChannelTag.tag_id, ChannelTag.channel_id).where(
|
||||
ChannelTag.user_id == user.id, ChannelTag.source == "user"
|
||||
owned_by(ChannelTag.user_id, user.id), ChannelTag.source == "user"
|
||||
)
|
||||
).all():
|
||||
if cid not in subscribed:
|
||||
|
||||
@@ -418,3 +418,48 @@ def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
|
||||
progress.report(probed, len(candidates), "probe")
|
||||
db.commit()
|
||||
return {"probed": probed, "shorts": shorts}
|
||||
|
||||
|
||||
def ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
|
||||
"""Make sure every given video id exists in the catalog, fetching + ingesting the
|
||||
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
|
||||
the API won't return are simply left out. The catalog's write for a playlist sync that names
|
||||
videos it has never seen; commits."""
|
||||
if not video_ids:
|
||||
return
|
||||
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
|
||||
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
|
||||
if not missing:
|
||||
return
|
||||
items = yt.get_videos(missing)
|
||||
chan_ids = {
|
||||
it.get("snippet", {}).get("channelId")
|
||||
for it in items
|
||||
if it.get("snippet", {}).get("channelId")
|
||||
}
|
||||
existing_ch = set(
|
||||
db.execute(select(Channel.id).where(Channel.id.in_(chan_ids))).scalars().all()
|
||||
)
|
||||
now = datetime.now(timezone.utc)
|
||||
for it in items:
|
||||
sn = it.get("snippet", {})
|
||||
cid = sn.get("channelId")
|
||||
if cid and cid not in existing_ch:
|
||||
db.add(Channel(id=cid, title=sn.get("channelTitle")))
|
||||
existing_ch.add(cid)
|
||||
db.flush()
|
||||
seen: set[str] = set()
|
||||
for it in items:
|
||||
vid = it.get("id")
|
||||
sn = it.get("snippet", {})
|
||||
cid = sn.get("channelId")
|
||||
if not vid or not cid or vid in seen:
|
||||
continue
|
||||
seen.add(vid)
|
||||
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
|
||||
# fallback here too so the constructor never carries an explicit NULL.
|
||||
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
|
||||
apply_video_details(v, it)
|
||||
v.enriched_at = now
|
||||
db.add(v)
|
||||
db.commit()
|
||||
|
||||
+251
-61
@@ -8,7 +8,8 @@ stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secr
|
||||
stored in the DB and stay env-only).
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Literal
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -27,6 +28,16 @@ _decrypt_warned: set[str] = set()
|
||||
_coerce_warned: set[tuple[str, str]] = set()
|
||||
|
||||
|
||||
# When a changed value takes effect. "live": the next read obeys it — every consumer reads the key
|
||||
# where it uses it, never once at import (test-held: a read outside a function fails). A value read
|
||||
# once at startup is live only when its writer also applies it (the job intervals reschedule, the log
|
||||
# level is set as it is saved). "restart": the process has to start
|
||||
# again. No key needs a restart today, and the Configuration page has no way to say so yet, so a
|
||||
# "restart" key is refused until the page can (test-held).
|
||||
Applies = Literal["live", "restart"]
|
||||
LIVE: Applies = "live"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConfigSpec:
|
||||
key: str
|
||||
@@ -37,6 +48,8 @@ class ConfigSpec:
|
||||
min: int | None = None
|
||||
max: int | None = None
|
||||
allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset"
|
||||
# Required, with no default: each key states it, so a new one cannot be live by omission.
|
||||
applies: Applies = field(kw_only=True)
|
||||
|
||||
@property
|
||||
def choices(self) -> tuple[str, ...]:
|
||||
@@ -45,30 +58,78 @@ class ConfigSpec:
|
||||
return choices_of(Settings.model_fields[self.default_attr].annotation)
|
||||
|
||||
|
||||
# Each interval job of the scheduler and the `settings` attribute holding its default period
|
||||
# (minutes). Its override is the `job_interval_<job>` key, set from the Scheduler page.
|
||||
JOB_INTERVAL_DEFAULTS: dict[str, str] = {
|
||||
"rss_poll": "rss_poll_minutes",
|
||||
"enrich": "enrich_interval_minutes",
|
||||
"backfill": "backfill_interval_minutes",
|
||||
"autotag": "autotag_interval_minutes",
|
||||
"shorts": "shorts_probe_interval_minutes",
|
||||
"subscriptions": "subscriptions_resync_minutes",
|
||||
"playlist_sync": "playlist_sync_minutes",
|
||||
"maintenance": "maintenance_interval_minutes",
|
||||
"explore_cleanup": "explore_cleanup_minutes",
|
||||
"download_gc": "download_gc_minutes",
|
||||
"plex_sync": "plex_sync_interval_min",
|
||||
"plex_watch_sync": "plex_watch_sync_interval_min",
|
||||
"plex_watch_reconcile": "plex_watch_reconcile_interval_min",
|
||||
"tmdb_enrich": "tmdb_enrich_interval_min",
|
||||
"people_enrich": "people_enrich_interval_min",
|
||||
"audit_gc": "audit_gc_minutes",
|
||||
"log_gc": "log_gc_minutes",
|
||||
}
|
||||
# Sane bounds for an admin-set interval (minutes).
|
||||
MIN_INTERVAL = 1
|
||||
MAX_INTERVAL = 1440 # one day
|
||||
|
||||
|
||||
def interval_key(job: str) -> str:
|
||||
return f"job_interval_{job}"
|
||||
|
||||
|
||||
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
|
||||
# off env into the admin UI; the admin page renders it automatically from this list. A group's keys
|
||||
# stay together, in the order its section lists them; a NEW group also needs its place in SECTIONS.
|
||||
SPECS: tuple[ConfigSpec, ...] = (
|
||||
# --- Access / registration ---
|
||||
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
|
||||
ConfigSpec("allow_registration", "bool", "access", "allow_registration", applies=LIVE),
|
||||
# --- Email / SMTP (one logical group; the password is the only secret) ---
|
||||
ConfigSpec("smtp_host", "str", "email", "smtp_host"),
|
||||
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535),
|
||||
ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True),
|
||||
ConfigSpec("smtp_from", "str", "email", "smtp_from"),
|
||||
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
|
||||
ConfigSpec("smtp_host", "str", "email", "smtp_host", applies=LIVE),
|
||||
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535, applies=LIVE),
|
||||
ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True, applies=LIVE),
|
||||
ConfigSpec("smtp_from", "str", "email", "smtp_from", applies=LIVE),
|
||||
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True, applies=LIVE),
|
||||
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
|
||||
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True),
|
||||
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True, applies=LIVE),
|
||||
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).
|
||||
ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True),
|
||||
ConfigSpec(
|
||||
"youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True, applies=LIVE
|
||||
),
|
||||
# --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the
|
||||
# install wizard / Configuration page so no restart is needed when the creds change. ---
|
||||
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True),
|
||||
ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True),
|
||||
# --- Quota (shared YouTube Data API daily budget) ---
|
||||
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
|
||||
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True, applies=LIVE),
|
||||
ConfigSpec(
|
||||
"backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000
|
||||
"google_client_secret", "str", "google", "google_client_secret", secret=True, applies=LIVE
|
||||
),
|
||||
# --- Quota (shared YouTube Data API daily budget) ---
|
||||
ConfigSpec(
|
||||
"quota_daily_budget",
|
||||
"int",
|
||||
"quota",
|
||||
"quota_daily_budget",
|
||||
min=1,
|
||||
max=1_000_000,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"backfill_quota_reserve",
|
||||
"int",
|
||||
"quota",
|
||||
"backfill_quota_reserve",
|
||||
min=0,
|
||||
max=1_000_000,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"search_daily_limit_per_user",
|
||||
@@ -77,10 +138,19 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
"search_daily_limit_per_user",
|
||||
min=0,
|
||||
max=100_000,
|
||||
applies=LIVE,
|
||||
),
|
||||
# Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
|
||||
ConfigSpec("search_source", "choice", "quota", "search_source"),
|
||||
ConfigSpec("quota_retention_days", "int", "quota", "quota_retention_days", min=0, max=3650),
|
||||
ConfigSpec("search_source", "choice", "quota", "search_source", applies=LIVE),
|
||||
ConfigSpec(
|
||||
"quota_retention_days",
|
||||
"int",
|
||||
"quota",
|
||||
"quota_retention_days",
|
||||
min=0,
|
||||
max=3650,
|
||||
applies=LIVE,
|
||||
),
|
||||
# --- Backfill (recent-first first pass per channel) ---
|
||||
ConfigSpec(
|
||||
"backfill_recent_max_videos",
|
||||
@@ -89,19 +159,46 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
"backfill_recent_max_videos",
|
||||
min=1,
|
||||
max=100_000,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500
|
||||
"backfill_recent_max_days",
|
||||
"int",
|
||||
"backfill",
|
||||
"backfill_recent_max_days",
|
||||
min=1,
|
||||
max=36_500,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168, applies=LIVE
|
||||
),
|
||||
ConfigSpec("rss_fallback_hours", "int", "backfill", "rss_fallback_hours", min=0, max=168),
|
||||
# --- Shorts probe ---
|
||||
ConfigSpec(
|
||||
"shorts_probe_max_seconds", "int", "shorts", "shorts_probe_max_seconds", min=1, max=3_600
|
||||
"shorts_probe_max_seconds",
|
||||
"int",
|
||||
"shorts",
|
||||
"shorts_probe_max_seconds",
|
||||
min=1,
|
||||
max=3_600,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000, applies=LIVE
|
||||
),
|
||||
ConfigSpec("shorts_probe_batch", "int", "shorts", "shorts_probe_batch", min=1, max=10_000),
|
||||
# --- Batch sizes ---
|
||||
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
|
||||
ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
|
||||
ConfigSpec(
|
||||
"enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50, applies=LIVE
|
||||
),
|
||||
ConfigSpec(
|
||||
"autotag_title_sample",
|
||||
"int",
|
||||
"batch",
|
||||
"autotag_title_sample",
|
||||
min=1,
|
||||
max=1_000,
|
||||
applies=LIVE,
|
||||
),
|
||||
# --- Maintenance (rolling re-validation of already-known videos) ---
|
||||
# Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin
|
||||
# knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard.
|
||||
@@ -112,16 +209,33 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
"maintenance_revalidate_batch",
|
||||
min=100,
|
||||
max=100_000,
|
||||
applies=LIVE,
|
||||
),
|
||||
# --- Discovery cleanup (search + explore ephemeral content) ---
|
||||
# Days an explored-but-unsubscribed channel is kept before the cleanup job purges it.
|
||||
ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650),
|
||||
# Days an un-kept live-search result video is kept before the cleanup job removes it.
|
||||
ConfigSpec("search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650),
|
||||
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
|
||||
ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0),
|
||||
ConfigSpec(
|
||||
"download_default_max_bytes", "int", "downloads", "download_default_max_bytes", min=0
|
||||
"explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650, applies=LIVE
|
||||
),
|
||||
# Days an un-kept live-search result video is kept before the cleanup job removes it.
|
||||
ConfigSpec(
|
||||
"search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650, applies=LIVE
|
||||
),
|
||||
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
|
||||
ConfigSpec(
|
||||
"download_total_max_bytes",
|
||||
"int",
|
||||
"downloads",
|
||||
"download_total_max_bytes",
|
||||
min=0,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"download_default_max_bytes",
|
||||
"int",
|
||||
"downloads",
|
||||
"download_default_max_bytes",
|
||||
min=0,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"download_default_max_concurrent",
|
||||
@@ -130,6 +244,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
"download_default_max_concurrent",
|
||||
min=1,
|
||||
max=20,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"download_default_max_jobs",
|
||||
@@ -138,6 +253,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
"download_default_max_jobs",
|
||||
min=1,
|
||||
max=100_000,
|
||||
applies=LIVE,
|
||||
),
|
||||
# min=1, NOT 0: a stored 0 used to read as "no per-file cap" (the cap was enforced as
|
||||
# `if cap and size > cap`, so a falsy value disabled it). Now the route passes this number
|
||||
@@ -161,14 +277,27 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
"download_upload_max_bytes",
|
||||
min=1,
|
||||
max=settings.download_upload_max_bytes,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650
|
||||
"download_retention_days",
|
||||
"int",
|
||||
"downloads",
|
||||
"download_retention_days",
|
||||
min=0,
|
||||
max=3_650,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec(
|
||||
"download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650
|
||||
"download_gc_grace_days",
|
||||
"int",
|
||||
"downloads",
|
||||
"download_gc_grace_days",
|
||||
min=0,
|
||||
max=3_650,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec("download_layout", "choice", "downloads", "download_layout"),
|
||||
ConfigSpec("download_layout", "choice", "downloads", "download_layout", applies=LIVE),
|
||||
ConfigSpec(
|
||||
"download_worker_concurrency",
|
||||
"int",
|
||||
@@ -176,60 +305,112 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
"download_worker_concurrency",
|
||||
min=1,
|
||||
max=16,
|
||||
applies=LIVE,
|
||||
),
|
||||
ConfigSpec("sponsorblock_default", "bool", "downloads", "sponsorblock_default"),
|
||||
# --- Audit log ---
|
||||
# Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever).
|
||||
ConfigSpec("audit_retention_days", "int", "audit", "audit_retention_days", min=0, max=3650),
|
||||
ConfigSpec(
|
||||
"audit_retention_days",
|
||||
"int",
|
||||
"audit",
|
||||
"audit_retention_days",
|
||||
min=0,
|
||||
max=3650,
|
||||
applies=LIVE,
|
||||
),
|
||||
# Days the job_runs table keeps a finished background run before the audit_gc job prunes it.
|
||||
ConfigSpec(
|
||||
"job_run_retention_days",
|
||||
"int",
|
||||
"audit",
|
||||
"job_run_retention_days",
|
||||
min=0,
|
||||
max=3650,
|
||||
applies=LIVE,
|
||||
),
|
||||
# --- Diagnostics ---
|
||||
# Runtime log level for the API process. Applied without a restart — see routes/config.py and
|
||||
# main.py's lifespan.
|
||||
ConfigSpec("log_level", "choice", "diagnostics", "log_level"),
|
||||
ConfigSpec("log_level", "choice", "diagnostics", "log_level", applies=LIVE),
|
||||
# Days the log_events table keeps a row (the log_gc job prunes). No "0 = forever": stored logs
|
||||
# carry account-linked request history, and the privacy policy names a limit.
|
||||
ConfigSpec("log_retention_days", "int", "diagnostics", "log_retention_days", min=1, max=90),
|
||||
ConfigSpec(
|
||||
"log_retention_days",
|
||||
"int",
|
||||
"diagnostics",
|
||||
"log_retention_days",
|
||||
min=1,
|
||||
max=90,
|
||||
applies=LIVE,
|
||||
),
|
||||
# --- Plex integration (optional; local-file playback + Plex metadata) ---
|
||||
ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled"),
|
||||
ConfigSpec("plex_server_url", "str", "plex", "plex_server_url"),
|
||||
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True),
|
||||
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"),
|
||||
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"),
|
||||
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"),
|
||||
# (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.)
|
||||
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
|
||||
ConfigSpec("plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64),
|
||||
ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled", applies=LIVE),
|
||||
ConfigSpec("plex_server_url", "str", "plex", "plex_server_url", applies=LIVE),
|
||||
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True, applies=LIVE),
|
||||
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map", applies=LIVE),
|
||||
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries", applies=LIVE),
|
||||
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map", applies=LIVE),
|
||||
ConfigSpec(
|
||||
"plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16, applies=LIVE
|
||||
),
|
||||
ConfigSpec(
|
||||
"plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64, applies=LIVE
|
||||
),
|
||||
# Transcode re-encode tuning. The libx264 knobs apply only to the software fallback.
|
||||
# crf 0–51 (x264 range); max_height 0 = source.
|
||||
ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw"),
|
||||
ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset"),
|
||||
ConfigSpec("plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51),
|
||||
ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw", applies=LIVE),
|
||||
ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset", applies=LIVE),
|
||||
ConfigSpec(
|
||||
"plex_transcode_max_height", "int", "plex", "plex_transcode_max_height", min=0, max=4320
|
||||
"plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51, applies=LIVE
|
||||
),
|
||||
ConfigSpec(
|
||||
"plex_transcode_max_height",
|
||||
"int",
|
||||
"plex",
|
||||
"plex_transcode_max_height",
|
||||
min=0,
|
||||
max=4320,
|
||||
applies=LIVE,
|
||||
),
|
||||
# Audio loudness normalisation on the HLS path (remux + transcode alike, not direct play).
|
||||
ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm"),
|
||||
# NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job
|
||||
# (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS.
|
||||
ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm", applies=LIVE),
|
||||
# --- TMDB enrichment (optional) — the API key is the only secret ---
|
||||
ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True),
|
||||
ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"),
|
||||
ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True, applies=LIVE),
|
||||
ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language", applies=LIVE),
|
||||
# --- Bug reports: the Gitea issue tracker a debug session's issue goes to ---
|
||||
ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url"),
|
||||
ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo"),
|
||||
ConfigSpec("issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True),
|
||||
# (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch
|
||||
# is a config.py default — both stay out of SPECS, like plex_sync_interval_min.)
|
||||
ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url", applies=LIVE),
|
||||
ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo", applies=LIVE),
|
||||
ConfigSpec(
|
||||
"issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True, applies=LIVE
|
||||
),
|
||||
# --- Scheduler: how often each interval job runs (minutes), one key per job ---
|
||||
*(
|
||||
ConfigSpec(
|
||||
interval_key(job),
|
||||
"int",
|
||||
"scheduler",
|
||||
attr,
|
||||
min=MIN_INTERVAL,
|
||||
max=MAX_INTERVAL,
|
||||
applies=LIVE,
|
||||
)
|
||||
for job, attr in JOB_INTERVAL_DEFAULTS.items()
|
||||
),
|
||||
)
|
||||
|
||||
# The admin page's sections (the owner's pick): sixteen groups as tabs wrapped to three rows at 133%
|
||||
# zoom, so they are a vertical list under four headings, in this order. Every group of SPECS sits in
|
||||
# exactly one section (test-held), so a new group cannot fall off the page.
|
||||
# exactly one section, or is edited on its own page (test-held), so a new group cannot fall off.
|
||||
SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("access", ("access", "google", "email")),
|
||||
("youtube", ("youtube", "quota", "backfill", "shorts", "batch", "explore")),
|
||||
("media", ("downloads", "plex", "tmdb")),
|
||||
("system", ("maintenance", "audit", "diagnostics", "issues")),
|
||||
)
|
||||
# Groups another admin page edits, so the Configuration page neither lists nor writes them: an
|
||||
# interval is changed on the Scheduler page, which also reschedules the running job — a write that
|
||||
# skipped it would leave the job on its old period while the page showed the new one.
|
||||
OWN_PAGE_GROUPS: tuple[str, ...] = ("scheduler",)
|
||||
|
||||
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}
|
||||
|
||||
@@ -238,6 +419,12 @@ def spec(key: str) -> ConfigSpec | None:
|
||||
return _BY_KEY.get(key)
|
||||
|
||||
|
||||
def config_page_spec(key: str) -> ConfigSpec | None:
|
||||
"""`spec`, for the Configuration page: None for a key edited on its own page as well."""
|
||||
s = _BY_KEY.get(key)
|
||||
return None if s is None or s.group in OWN_PAGE_GROUPS else s
|
||||
|
||||
|
||||
def secrets_manageable() -> bool:
|
||||
"""Secret values can only be stored when Fernet is configured (TOKEN_ENCRYPTION_KEY)."""
|
||||
return bool(settings.token_encryption_key)
|
||||
@@ -426,13 +613,16 @@ def reset(db: Session, key: str) -> None:
|
||||
|
||||
|
||||
def describe(db: Session) -> dict:
|
||||
"""Admin-UI metadata + current values, grouped. Secret values are never echoed —
|
||||
only whether an override is set and whether an env default exists."""
|
||||
"""Admin-UI metadata + current values, grouped — the keys of the Configuration page, not those
|
||||
edited on their own page. Secret values are never echoed — only whether an override is set and
|
||||
whether an env default exists."""
|
||||
groups: dict[str, list[dict]] = {}
|
||||
# Every override in one query: a db.get per key was one round trip each (two for a key with no
|
||||
# row, which the session cannot cache), ~40 ms of the page's ~55 on prod.
|
||||
rows = {r.key: r for r in db.scalars(select(SystemConfig))}
|
||||
for s in SPECS:
|
||||
if s.group in OWN_PAGE_GROUPS:
|
||||
continue
|
||||
row = rows.get(s.key)
|
||||
is_set = row is not None
|
||||
item: dict = {
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Every table, registered on `Base.metadata`.
|
||||
|
||||
Each unit declares its tables in its own `models` module, so importing `app.models` alone no longer
|
||||
names the whole schema. Whatever needs all of it — the migrations, the test database, a check over
|
||||
every mapper — imports this module.
|
||||
"""
|
||||
|
||||
import app.downloads.models # Capture's tables
|
||||
import app.models # the core tables
|
||||
import app.reader.models # noqa: F401 — Reader's tables
|
||||
@@ -1,13 +1,54 @@
|
||||
"""Shared "who is an active, non-suspended account" predicate.
|
||||
"""Whose a row is, and who an account is to the others — one place for both.
|
||||
|
||||
Used wherever a user must be filtered out of a recipient list or rejected as a target: the download
|
||||
share-by-email and recipient-picker endpoints. Kept in one place so the two forms (SQL clauses for a
|
||||
query, Python check for a loaded row) can't drift apart, and so a route module doesn't have to import
|
||||
another route module just to reuse the rule.
|
||||
**Ownership.** A user-owned table says whose a row is in one column (`user_id`, or `to_user_id` for
|
||||
something handed to them). A unit's reads and writes name that column through `owned_by` and fetch a
|
||||
row by id through `owned_row`, never with an inline `Model.user_id == …`: the rule "a row owned by
|
||||
someone else is answered like a missing one" then lives here, and a contract test
|
||||
(`test_unit_fences.py`) holds a switched-on unit to it.
|
||||
|
||||
**Recipients.** "An active, non-suspended account" — the download share-by-email and the recipient
|
||||
picker filter on it. Kept here so the two forms (SQL clauses for a query, a Python check for a loaded
|
||||
row) can't drift apart.
|
||||
"""
|
||||
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from sqlalchemy import ColumnElement, func, select
|
||||
from sqlalchemy.orm import InstrumentedAttribute, Session
|
||||
|
||||
from app.collation import collated
|
||||
from app.db import Base
|
||||
from app.models import User
|
||||
|
||||
M = TypeVar("M", bound=Base)
|
||||
|
||||
|
||||
def _owner_id(owner: User | int) -> int:
|
||||
return owner if isinstance(owner, int) else owner.id
|
||||
|
||||
|
||||
def owned_by(column: InstrumentedAttribute, owner: User | int) -> ColumnElement[bool]:
|
||||
"""`column` — a table's owner column — names `owner`."""
|
||||
return column == _owner_id(owner)
|
||||
|
||||
|
||||
def owned_row(
|
||||
db: Session,
|
||||
model: type[M],
|
||||
id_: Any,
|
||||
owner: User | int,
|
||||
*,
|
||||
owner_attr: str = "user_id",
|
||||
lock: bool = False,
|
||||
) -> M | None:
|
||||
"""The row of `model` with this primary key when `owner` owns it, else None — a row of someone
|
||||
else's is the same answer as a missing one, so a caller can never tell the two apart. `lock`
|
||||
takes it FOR UPDATE (past the identity map)."""
|
||||
row = db.get(model, id_, with_for_update=True if lock else None)
|
||||
if row is None or getattr(row, owner_attr) != _owner_id(owner):
|
||||
return None
|
||||
return row
|
||||
|
||||
|
||||
def messageable_clauses() -> list:
|
||||
"""SQL clauses for "an active, non-suspended account" — splat into a recipient query."""
|
||||
@@ -20,3 +61,30 @@ def messageable_clauses() -> list:
|
||||
def is_messageable_user(u: User | None) -> bool:
|
||||
"""Python mirror of `messageable_clauses()` for an already-loaded row (a share recipient)."""
|
||||
return bool(u and u.is_active and not u.is_suspended)
|
||||
|
||||
|
||||
def user_by_email(db: Session, email: str) -> User | None:
|
||||
"""The account with this address, matched without regard to case."""
|
||||
return db.execute(
|
||||
select(User).where(func.lower(User.email) == email.strip().lower())
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def messageable_others(db: Session, user: User, lang: str) -> list[User]:
|
||||
"""Every account `user` may hand something to: active, not suspended, not themselves; in the
|
||||
reader's alphabet by address."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(User)
|
||||
.where(User.id != user.id, *messageable_clauses())
|
||||
.order_by(collated(User.email, lang))
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def emails_of(db: Session, ids: set[int] | None = None) -> dict[int, str]:
|
||||
"""Each account's address by id, in address order: these ids, or every account."""
|
||||
stmt = select(User.id, User.email).order_by(User.email)
|
||||
if ids is not None:
|
||||
stmt = stmt.where(User.id.in_(ids))
|
||||
return dict(db.execute(stmt).tuples().all())
|
||||
|
||||
@@ -46,7 +46,8 @@ from app.downloads.jobdb import (
|
||||
_staging_dir,
|
||||
)
|
||||
from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_stop = threading.Event()
|
||||
|
||||
@@ -25,7 +25,7 @@ from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine, event, text
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
import app.models # register every model on Base.metadata
|
||||
import app.tables # every table on Base.metadata
|
||||
import app.state as app_state
|
||||
from app import apitokens, security
|
||||
from app.auth import current_user
|
||||
|
||||
@@ -106,7 +106,7 @@ def test_the_single_playlist_repull_shares_the_pull_cooldown(db, as_user, make_u
|
||||
# SEC-14, the sweep half: `revert-youtube` spends under the SAME QuotaAction.PLAYLISTS_PULL key
|
||||
# as the bulk `sync-youtube`, so limiting only one of them left the identical drain open one
|
||||
# route over. Both must draw on the one allowance.
|
||||
from app.models import Playlist
|
||||
from app.reader.models import Playlist
|
||||
from app.routes import playlists as plroutes
|
||||
|
||||
user = make_user()
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
"""The activity page's reads: the audit log and the background runs, filtered on the server and
|
||||
paged by a keyset, and the quota breakdown over a time range and one account.
|
||||
|
||||
Each list is asked for the rows a filter must keep AND must drop, and a page boundary is crossed
|
||||
with rows that share a timestamp — the case an offset or a timestamp-only cursor gets wrong.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from app.auth import admin_user
|
||||
from app.main import app
|
||||
from app.models import AuditLog, Channel, JobRun, LogEvent, QuotaEvent
|
||||
from tests.test_body_limit import _route_objects
|
||||
|
||||
T0 = datetime(2026, 10, 1, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _audit(db, action, actor=None, at=T0, summary=None, request_id=None) -> int:
|
||||
row = AuditLog(
|
||||
actor_id=actor, action=action, summary=summary, created_at=at, request_id=request_id
|
||||
)
|
||||
db.add(row)
|
||||
db.flush()
|
||||
return row.id
|
||||
|
||||
|
||||
def _run(db, job, status="ok", actor=None, at=T0, target=None, trigger="scheduled") -> int:
|
||||
row = JobRun(
|
||||
job=job,
|
||||
trigger=trigger,
|
||||
actor_id=actor,
|
||||
target=target,
|
||||
status=status,
|
||||
started_at=at,
|
||||
finished_at=None if status == "running" else at + timedelta(seconds=4),
|
||||
quota_units=0,
|
||||
)
|
||||
db.add(row)
|
||||
db.flush()
|
||||
return row.id
|
||||
|
||||
|
||||
def _ids(body) -> list[int]:
|
||||
return [r["id"] for r in body["items"]]
|
||||
|
||||
|
||||
def test_the_audit_list_filters_on_the_server_and_pages_across_a_shared_timestamp(
|
||||
db, make_user, as_user
|
||||
):
|
||||
admin = make_user(role="admin")
|
||||
a = _audit(db, "config_set", admin.id, T0, "quota_daily_budget = 9000")
|
||||
b = _audit(db, "config_set", admin.id, T0, "log_level = DEBUG")
|
||||
c = _audit(db, "config_set", admin.id, T0, "smtp_host = mail")
|
||||
bg = _audit(db, "user_purged", None, T0 - timedelta(days=3))
|
||||
db.commit()
|
||||
client = as_user(admin)
|
||||
|
||||
first = client.get("/api/admin/audit", params={"limit": 2, "action": "config_set"}).json()
|
||||
assert _ids(first) == [c, b]
|
||||
second = client.get(
|
||||
"/api/admin/audit",
|
||||
params={"limit": 2, "action": "config_set", "before": first["next_cursor"]},
|
||||
).json()
|
||||
assert (_ids(second), second["next_cursor"]) == ([a], None)
|
||||
# A page that ends exactly at the last row says there is nothing older.
|
||||
whole = client.get("/api/admin/audit", params={"limit": 3, "action": "config_set"}).json()
|
||||
assert (_ids(whole), whole["next_cursor"]) == ([c, b, a], None)
|
||||
assert first["accounts"] == {str(admin.id): admin.email}
|
||||
|
||||
def ids(**params) -> list[int]:
|
||||
return _ids(client.get("/api/admin/audit", params=params).json())
|
||||
|
||||
assert ids(user="none") == [bg]
|
||||
assert ids(user=str(admin.id)) == [c, b, a]
|
||||
assert ids(q="DEBUG") == [b]
|
||||
assert ids(since=(T0 - timedelta(days=1)).isoformat()) == [c, b, a]
|
||||
assert ids(until=(T0 - timedelta(days=1)).isoformat()) == [bg]
|
||||
# The action menu counts every action under the other filters, its own pick included.
|
||||
facets = client.get("/api/admin/audit/facets", params={"action": "config_set"}).json()
|
||||
assert facets["actions"] == [
|
||||
{"value": "config_set", "count": 3},
|
||||
{"value": "user_purged", "count": 1},
|
||||
]
|
||||
|
||||
|
||||
def test_an_admin_action_keeps_the_request_it_was_made_in(db, make_user, as_user):
|
||||
client = as_user(make_user(role="admin"))
|
||||
r = client.patch("/api/admin/config/quota_daily_budget", json={"value": 9500})
|
||||
assert r.status_code == 200, r.text
|
||||
(row,) = client.get("/api/admin/audit", params={"action": "config_set"}).json()["items"]
|
||||
assert row["request_id"] == r.headers["x-request-id"]
|
||||
|
||||
|
||||
def test_an_admin_action_says_how_many_of_its_requests_lines_are_still_kept(db, make_user, as_user):
|
||||
"""A request's lines are kept days, its audit row months: a link to lines gone would open an
|
||||
empty list, so the row says how many are left, as a run does."""
|
||||
admin = make_user(role="admin")
|
||||
kept = _audit(db, "config_set", actor=admin.id, request_id="req-kept-0001")
|
||||
gone = _audit(db, "config_set", actor=admin.id, request_id="req-gone-0001")
|
||||
db.add_all(
|
||||
[
|
||||
LogEvent(
|
||||
ts=T0,
|
||||
level="INFO",
|
||||
sev=20,
|
||||
source="api",
|
||||
logger="t",
|
||||
msg="m",
|
||||
request_id="req-kept-0001",
|
||||
)
|
||||
for _ in range(3)
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
items = as_user(admin).get("/api/admin/audit").json()["items"]
|
||||
lines = {r["id"]: r["log_lines"] for r in items}
|
||||
assert (lines[kept], lines[gone]) == (3, 0)
|
||||
|
||||
|
||||
def test_the_run_list_filters_pages_and_names_the_channel_and_the_log_lines(db, make_user, as_user):
|
||||
admin = make_user(role="admin")
|
||||
db.add(Channel(id="UCchan", title="Aileen Wang"))
|
||||
ok1 = _run(db, "rss_poll", at=T0)
|
||||
ok2 = _run(db, "rss_poll", at=T0)
|
||||
bad = _run(db, "backfill", "error", admin.id, T0, trigger="manual")
|
||||
adhoc = _run(
|
||||
db, "channel_backfill", "ok", admin.id, T0 - timedelta(hours=1), "UCchan", "ad_hoc"
|
||||
)
|
||||
for _ in range(2):
|
||||
db.add(
|
||||
LogEvent(ts=T0, level="WARNING", sev=30, source="api", logger="t", msg="m", run_id=bad)
|
||||
)
|
||||
db.commit()
|
||||
client = as_user(admin)
|
||||
|
||||
first = client.get("/api/admin/runs", params={"limit": 2}).json()
|
||||
rest = client.get("/api/admin/runs", params={"limit": 2, "before": first["next_cursor"]}).json()
|
||||
assert _ids(first) + _ids(rest) == [bad, ok2, ok1, adhoc]
|
||||
assert rest["next_cursor"] is None
|
||||
rows = {r["id"]: r for r in first["items"] + rest["items"]}
|
||||
assert (rows[bad]["log_lines"], rows[ok2]["log_lines"]) == (2, 0)
|
||||
assert (rows[adhoc]["target_title"], rows[ok1]["target_title"]) == ("Aileen Wang", None)
|
||||
|
||||
def ids(**params) -> list[int]:
|
||||
return _ids(client.get("/api/admin/runs", params=params).json())
|
||||
|
||||
assert ids(status="error") == [bad]
|
||||
assert ids(job="rss_poll") == [ok2, ok1]
|
||||
assert ids(user="none") == [ok2, ok1]
|
||||
assert ids(user=str(admin.id)) == [bad, adhoc]
|
||||
assert ids(q="UCchan") == [adhoc]
|
||||
assert ids(since=(T0 - timedelta(minutes=30)).isoformat()) == [bad, ok2, ok1]
|
||||
assert client.get("/api/admin/runs", params={"status": "finished"}).status_code == 422
|
||||
|
||||
facets = client.get("/api/admin/runs/facets", params={"status": "error"}).json()
|
||||
assert facets["statuses"] == [{"value": "ok", "count": 3}, {"value": "error", "count": 1}]
|
||||
assert facets["jobs"] == [{"value": "backfill", "count": 1}]
|
||||
assert facets["total"] == 4
|
||||
|
||||
|
||||
def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_user):
|
||||
admin = make_user(role="admin")
|
||||
now = datetime.now(timezone.utc)
|
||||
db.add_all(
|
||||
[
|
||||
QuotaEvent(user_id=admin.id, action="search", units=100, created_at=now),
|
||||
QuotaEvent(user_id=None, action="backfill", units=7, created_at=now),
|
||||
QuotaEvent(
|
||||
user_id=admin.id, action="search", units=50, created_at=now - timedelta(days=10)
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
client = as_user(admin)
|
||||
|
||||
def totals(**params) -> dict:
|
||||
body = client.get("/api/quota/admin", params=params).json()
|
||||
# The chart is the same filter's: its days add up to the table's total.
|
||||
assert sum(d["total"] for d in body["daily"]) == sum(r["total"] for r in body["rows"])
|
||||
return {r["user_id"]: r["total"] for r in body["rows"]}
|
||||
|
||||
assert totals() == {admin.id: 150, None: 7} # the last 30 days by default
|
||||
assert totals(since=(now - timedelta(days=1)).isoformat()) == {admin.id: 100, None: 7}
|
||||
assert totals(until=(now - timedelta(days=1)).isoformat()) == {admin.id: 50}
|
||||
assert totals(user="none") == {None: 7}
|
||||
assert totals(user=str(admin.id)) == {admin.id: 150}
|
||||
|
||||
|
||||
def test_every_admin_read_and_write_is_admin_only():
|
||||
"""The route table, not a list kept by hand: every /api/admin route and the quota breakdown
|
||||
resolve `admin_user`, so a dropped dependency cannot hand an account's runs, errors and e-mail
|
||||
addresses to any signed-in user."""
|
||||
routes = _route_objects(app)
|
||||
admin = {k: r for k, r in routes.items() if k[1].startswith(("/api/admin", "/api/quota/admin"))}
|
||||
assert ("GET", "/api/admin/runs") in admin and ("GET", "/api/quota/admin") in admin
|
||||
open_ = []
|
||||
for key, route in admin.items():
|
||||
stack, calls = [route.dependant], set()
|
||||
while stack:
|
||||
d = stack.pop()
|
||||
calls.add(d.call)
|
||||
stack.extend(d.dependencies)
|
||||
if admin_user not in calls:
|
||||
open_.append(key)
|
||||
assert open_ == []
|
||||
@@ -24,6 +24,7 @@ def _row(**kw) -> dict:
|
||||
"request_id": None,
|
||||
"user_id": None,
|
||||
"job_id": None,
|
||||
"run_id": None,
|
||||
"version": "0.90.0",
|
||||
}
|
||||
row.update(kw)
|
||||
@@ -80,6 +81,7 @@ def test_newest_first_and_every_row_once_across_pages_even_on_equal_timestamps(d
|
||||
({"event": "none"}, {"warn"}),
|
||||
({"request_id": "req-abcdef12"}, {"mine"}),
|
||||
({"job_id": 42}, {"err"}),
|
||||
({"run_id": 7}, {"warn"}), # a background run's lines, not a download job's
|
||||
({"q": "FORBIDDEN"}, {"err"}), # in attrs, any case
|
||||
({"q": "KeyError"}, {"warn"}), # in the stack
|
||||
({"q": "100%"}, {"plain"}), # a literal, not a wildcard
|
||||
@@ -102,7 +104,14 @@ def test_each_filter_selects_exactly_its_rows(db, admin, make_user, params, expe
|
||||
job_id=42,
|
||||
attrs={"code": "forbidden"},
|
||||
),
|
||||
_row(msg="warn", level="WARNING", sev=13, event=None, exc="Traceback …\nKeyError: 'x'"),
|
||||
_row(
|
||||
msg="warn",
|
||||
level="WARNING",
|
||||
sev=13,
|
||||
event=None,
|
||||
exc="Traceback …\nKeyError: 'x'",
|
||||
run_id=7,
|
||||
),
|
||||
_row(msg="plain", ts=T0 - timedelta(hours=2), attrs={"note": "100% done"}),
|
||||
# Matched by no filter above but `until`: another account, and `100%` read as a pattern.
|
||||
_row(msg="decoy", ts=T0 - timedelta(hours=3), attrs={"note": "1000 done"}, user_id=99),
|
||||
|
||||
@@ -25,16 +25,8 @@ from app.config import settings
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.main import app
|
||||
from app.notifications import create_notification
|
||||
from app.models import (
|
||||
ApiToken,
|
||||
ApiTokenGrant,
|
||||
AuditLog,
|
||||
DownloadJob,
|
||||
FileFolder,
|
||||
MediaAsset,
|
||||
Notification,
|
||||
User,
|
||||
)
|
||||
from app.models import ApiToken, ApiTokenGrant, AuditLog, Notification, User
|
||||
from app.downloads.models import DownloadJob, FileFolder, MediaAsset
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes import tokens as tokens_routes
|
||||
from app.routes.auth import bump_session_epoch
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""An admin action that counts something keeps the count as data in its audit row's `after`, under
|
||||
the key the audit log names in the reader's language (`scheduler.result.<key>`, frontend
|
||||
auditColumns.tsx) — not only in the English `summary`. A renamed key would fall back to the raw word.
|
||||
lib/jobResult.ts, called from lib/auditText.ts) — not only in the English `summary`. A renamed key would fall back to the raw word.
|
||||
DB lane (Postgres).
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""sync/autotag's orphan cleanup (the module was 32% covered).
|
||||
"""The auto-tag job's orphan cleanup (`reader.service.prune_orphan_system_tags`).
|
||||
|
||||
A system tag no channel carries any more is deleted — unless a saved feed view filters on it. The
|
||||
views hold tag ids; a deleted tag returns under a NEW id when a channel earns it again, and the feed
|
||||
@@ -7,8 +7,9 @@ drops an unknown id, so the view silently showed the whole feed under its old na
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import Channel, ChannelTag, SavedView, Tag
|
||||
from app.sync import autotag
|
||||
from app.models import Channel
|
||||
from app.reader.models import ChannelTag, SavedView, Tag
|
||||
from app.reader import service as reader_service
|
||||
|
||||
|
||||
def test_an_orphan_system_tag_goes_but_one_a_saved_view_filters_on_stays(db, make_user):
|
||||
@@ -30,7 +31,8 @@ def test_an_orphan_system_tag_goes_but_one_a_saved_view_filters_on_stays(db, mak
|
||||
db.add(SavedView(user_id=user.id, source="plex", name="P", filters={"genres": ["Drama"]}))
|
||||
db.commit()
|
||||
|
||||
assert autotag._cleanup_orphan_system_tags(db) == 1
|
||||
assert reader_service.prune_orphan_system_tags(db) == 1
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
names = set(db.execute(select(Tag.name)).scalars())
|
||||
assert names == {"Gaming", "Music", "Mine"} # only the unreferenced system orphan went
|
||||
@@ -48,7 +50,8 @@ def test_a_malformed_saved_view_cannot_break_the_cleanup(db, make_user):
|
||||
):
|
||||
db.add(SavedView(user_id=user.id, source="feed", name=f"bad{i}", filters=filters))
|
||||
db.commit()
|
||||
assert autotag._cleanup_orphan_system_tags(db) == 1
|
||||
assert reader_service.prune_orphan_system_tags(db) == 1
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
assert db.execute(select(Tag).where(Tag.name == "Cooking")).first() is None
|
||||
|
||||
@@ -61,4 +64,5 @@ def test_a_view_kept_tag_list_larger_than_a_bind_parameter_limit_still_cleans_up
|
||||
huge = list(range(10_000_000, 10_070_000)) # past psycopg's 65535 bind parameters
|
||||
db.add(SavedView(user_id=user.id, source="feed", name="huge", filters={"tags": huge}))
|
||||
db.commit()
|
||||
assert autotag._cleanup_orphan_system_tags(db) == 1
|
||||
assert reader_service.prune_orphan_system_tags(db) == 1
|
||||
db.commit()
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
"""Capture's service and queries, one use case each, below the HTTP layer.
|
||||
|
||||
The routes are thin callers now, and their own tests stay the contract a client sees. These hold
|
||||
what each write does and leaves on its own: that it flushes and leaves the commit to its caller,
|
||||
that a row of someone else's reads as a missing one, and the rule each write carries (a hash kept
|
||||
once, a password change that retires the old grants, a quota floored). DB-backed (real Postgres).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.audit import AuditAction
|
||||
from app.downloads import folders, links, queries, quota, service
|
||||
from app.downloads.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
DownloadQuota,
|
||||
DownloadShare,
|
||||
FileFolder,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
)
|
||||
from app.models import AuditLog, Notification
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
|
||||
def _file(db, user, *, sha256=None) -> DownloadJob:
|
||||
asset = MediaAsset(
|
||||
source_kind="url",
|
||||
source_ref=f"https://example.com/{user.id}.mp4",
|
||||
format_sig=f"sig-{user.id}",
|
||||
status="ready",
|
||||
rel_path=f"f-{user.id}.mp4",
|
||||
ref_count=1,
|
||||
media_kind="video",
|
||||
container="mp4",
|
||||
sha256=sha256,
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
job = DownloadJob(
|
||||
user_id=user.id,
|
||||
asset_id=asset.id,
|
||||
source_kind="url",
|
||||
source_ref=asset.source_ref,
|
||||
status="done",
|
||||
profile_snapshot={},
|
||||
display_name="Clip",
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
return job
|
||||
|
||||
|
||||
def _uncommitted(db):
|
||||
"""What the caller still has to commit: the session holds pending work."""
|
||||
return bool(db.new or db.dirty or db.deleted) or db.in_transaction()
|
||||
|
||||
|
||||
# --- whose a row is ------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_row_of_someone_elses_reads_as_a_missing_one(db, make_user):
|
||||
owner, stranger = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert owned_row(db, DownloadJob, job.id, owner) is job
|
||||
assert owned_row(db, DownloadJob, job.id, stranger) is None
|
||||
assert owned_row(db, DownloadJob, job.id + 1000, owner) is None
|
||||
mine = db.execute(select(DownloadJob.id).where(owned_by(DownloadJob.user_id, owner.id)))
|
||||
assert mine.scalars().all() == [job.id]
|
||||
|
||||
|
||||
def test_a_file_is_readable_by_its_owner_and_whoever_it_was_shared_with(db, make_user):
|
||||
owner, friend, stranger = make_user(), make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert service.share(db, job, owner, friend) is True
|
||||
db.commit()
|
||||
assert queries.accessible_job(db, owner, job.id) is job
|
||||
assert queries.accessible_job(db, friend, job.id) is job
|
||||
assert queries.accessible_job(db, stranger, job.id) is None
|
||||
assert queries.own_job(db, friend, job.id) is None # shared is not owned
|
||||
|
||||
|
||||
def test_the_tray_and_the_library_list_only_the_callers_own_files(db, make_user):
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
mine, theirs = make_user(), make_user()
|
||||
own, other = _file(db, mine), _file(db, theirs)
|
||||
since = datetime.now(timezone.utc) - timedelta(days=1)
|
||||
for job in (own, other):
|
||||
job.ready_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
assert [j.id for j in queries.queue(db, mine, since)] == [own.id]
|
||||
library = queries.library(
|
||||
db,
|
||||
mine,
|
||||
folder_id=None,
|
||||
listable=None,
|
||||
shared=False,
|
||||
term="",
|
||||
sort="added",
|
||||
order="desc",
|
||||
lang="en",
|
||||
)
|
||||
assert [j.id for j in library] == [own.id]
|
||||
assert [r for r, _ in queries.statuses_by_source(db, mine, ["done"])] == [own.source_ref]
|
||||
|
||||
|
||||
# --- locks -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_job_lock_answers_none_for_a_job_not_the_callers(db, make_user):
|
||||
owner, stranger = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert service.lock_own_job(db, stranger, job.id) is None
|
||||
assert service.lock_own_job(db, owner, job.id) is job
|
||||
assert service.hold_for_share(db, stranger, job.id) is None
|
||||
assert service.hold_for_share(db, owner, job.id) is job
|
||||
|
||||
|
||||
# --- profiles --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_profile_is_made_after_the_users_others_and_only_theirs_to_change(db, make_user):
|
||||
user, other = make_user(), make_user()
|
||||
first = service.create_profile(db, user, "x" * 100, {"kind": "video"})
|
||||
second = service.create_profile(db, user, "Audio", {"kind": "audio"})
|
||||
assert _uncommitted(db)
|
||||
db.commit()
|
||||
assert first.name == "x" * 80 # names are capped
|
||||
assert second.sort_order == first.sort_order + 1
|
||||
assert queries.own_profile(db, other, first.id) is None
|
||||
service.update_profile(first, name="Video", spec=None)
|
||||
db.commit()
|
||||
assert (first.name, first.spec) == ("Video", {"kind": "video"})
|
||||
service.delete_profile(db, first)
|
||||
db.commit()
|
||||
assert [p.id for p in queries.profiles_for(db, user)] == [second.id]
|
||||
|
||||
|
||||
# --- resume points ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_resume_point_is_one_row_per_user_and_file_and_a_zero_clears_it(db, make_user):
|
||||
user = make_user()
|
||||
job = _file(db, user)
|
||||
service.set_position(db, user, job.id, 40)
|
||||
service.set_position(db, user, job.id, 90)
|
||||
db.commit()
|
||||
rows = db.execute(select(LibraryState.position_seconds)).scalars().all()
|
||||
assert rows == [90]
|
||||
assert queries.positions_for(db, user, [job]) == {job.id: 90}
|
||||
service.set_position(db, user, job.id, 0)
|
||||
service.set_position(db, user, job.id, 0) # the beacon's second clear deletes nothing
|
||||
db.commit()
|
||||
assert db.execute(select(LibraryState)).first() is None
|
||||
|
||||
|
||||
# --- shares ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_share_is_made_once_with_one_notice_and_undone_by_either_side(db, make_user):
|
||||
owner, friend = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
assert service.share(db, job, owner, friend) is True
|
||||
assert service.share(db, job, owner, friend) is False
|
||||
assert _uncommitted(db)
|
||||
db.commit()
|
||||
notices = db.execute(select(Notification).where(Notification.user_id == friend.id)).scalars()
|
||||
assert [n.data["job_id"] for n in notices] == [job.id]
|
||||
assert service.unshare(db, job.id, friend) is True
|
||||
assert service.unshare(db, job.id, friend) is False
|
||||
db.commit()
|
||||
assert db.execute(select(DownloadShare)).first() is None
|
||||
|
||||
|
||||
# --- a file's display fields, its access clock, its hash ------------------------------------
|
||||
|
||||
|
||||
def test_only_display_fields_can_be_set_on_a_file(db, make_user):
|
||||
job = _file(db, make_user())
|
||||
service.set_display(job, display_name="New", extra_links=None)
|
||||
assert job.display_name == "New"
|
||||
with pytest.raises(ValueError):
|
||||
service.set_display(job, status="error")
|
||||
assert job.status == "done"
|
||||
|
||||
|
||||
def test_a_hash_is_kept_once_and_never_overwritten(db, make_user):
|
||||
job = _file(db, make_user())
|
||||
service.store_sha256(db, job.asset_id, "a" * 64)
|
||||
service.store_sha256(db, job.asset_id, "b" * 64)
|
||||
db.commit()
|
||||
assert db.get(MediaAsset, job.asset_id).sha256 == "a" * 64
|
||||
|
||||
|
||||
def test_serving_a_file_restarts_its_idle_clock(db, make_user):
|
||||
job = _file(db, make_user())
|
||||
asset = queries.asset_of(db, job)
|
||||
before = asset.last_access_at
|
||||
service.touch(asset)
|
||||
assert asset.last_access_at is not None and asset.last_access_at != before
|
||||
|
||||
|
||||
# --- links -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _actions(db) -> list[str]:
|
||||
return db.execute(select(AuditLog.action).order_by(AuditLog.id)).scalars().all()
|
||||
|
||||
|
||||
def test_a_link_is_made_changed_and_revoked_with_an_audit_row_each(db, make_user):
|
||||
user = make_user()
|
||||
job = _file(db, user)
|
||||
link, token = links.create(
|
||||
db, user, password="pw", allow_download=False, expires_days=3, job_id=job.id
|
||||
)
|
||||
db.commit()
|
||||
assert links.find(db, token) is link
|
||||
assert link.expires_at is not None and link.password_hash is not None
|
||||
version = link.password_version
|
||||
links.update(db, user, link, password="other", sent=frozenset({"password"}))
|
||||
db.commit()
|
||||
assert link.password_version == version + 1 # the old grants stop working
|
||||
links.update(db, user, link, allow_download=True, sent=frozenset()) # nothing was sent
|
||||
assert link.allow_download is False
|
||||
assert [link.id for link in queries.links_to(db, job_id=job.id)] == [link.id]
|
||||
links.revoke(db, user, link)
|
||||
db.commit()
|
||||
assert db.execute(select(DownloadLink)).first() is None
|
||||
assert _actions(db) == [
|
||||
AuditAction.DOWNLOAD_LINK_CREATE,
|
||||
AuditAction.DOWNLOAD_LINK_UPDATE,
|
||||
AuditAction.DOWNLOAD_LINK_REVOKE,
|
||||
]
|
||||
|
||||
|
||||
def test_a_link_is_the_callers_only_through_a_file_or_folder_of_theirs(db, make_user):
|
||||
owner, stranger = make_user(), make_user()
|
||||
job = _file(db, owner)
|
||||
link, _ = links.create(
|
||||
db, owner, password="", allow_download=True, expires_days=None, job_id=job.id
|
||||
)
|
||||
db.commit()
|
||||
assert queries.own_link(db, owner, link.id) is link
|
||||
assert queries.own_link(db, stranger, link.id) is None
|
||||
|
||||
|
||||
def test_a_view_is_counted_in_sql(db, make_user):
|
||||
user = make_user()
|
||||
job = _file(db, user)
|
||||
link, _ = links.create(
|
||||
db, user, password="", allow_download=True, expires_days=None, job_id=job.id
|
||||
)
|
||||
db.commit()
|
||||
links.count_view(db, link)
|
||||
links.count_view(db, link)
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
assert db.get(DownloadLink, link.id).view_count == 2
|
||||
|
||||
|
||||
# --- folders ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_folder_name_is_one_per_user_and_a_source_folder_keeps_its_own(db, make_user):
|
||||
user, other = make_user(), make_user()
|
||||
mine = folders.create_own(db, user.id, "Trips")
|
||||
with pytest.raises(folders.FolderError, match="file_folder_name_taken"):
|
||||
folders.create_own(db, user.id, "trips")
|
||||
folders.create_own(db, other.id, "Trips") # another user's namesake is theirs
|
||||
db.commit()
|
||||
source = folders.source_folder(db, user.id, folders.SourceKey.UPLOAD)
|
||||
with pytest.raises(folders.FolderError, match="file_folder_not_own"):
|
||||
folders.rename(db, source, "Mine now")
|
||||
folders.rename(db, mine, "Holidays")
|
||||
db.commit()
|
||||
assert db.get(FileFolder, mine.id).name == "Holidays"
|
||||
assert queries.own_folder(db, other, mine.id) is None
|
||||
|
||||
|
||||
# --- quota -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_custom_quota_starts_from_what_applies_and_floors_each_value(db, make_user):
|
||||
user = make_user()
|
||||
defaults = quota.resolve(db, user.id)
|
||||
quota.set_custom(db, user.id, max_concurrent=0, max_bytes=-5)
|
||||
db.commit()
|
||||
row = queries.custom_quota(db, user.id)
|
||||
assert (row.max_concurrent, row.max_bytes, row.max_jobs) == (1, 0, defaults.max_jobs)
|
||||
assert quota.reset_custom(db, user.id) is True
|
||||
assert quota.reset_custom(db, user.id) is False
|
||||
db.commit()
|
||||
assert db.get(DownloadQuota, user.id) is None
|
||||
@@ -107,25 +107,48 @@ def _setup_channel_and_sub(db, *, email):
|
||||
|
||||
|
||||
def _patch_deep_requested(db, monkeypatch, override_dep, user):
|
||||
submitted: list[tuple] = []
|
||||
calls: list[tuple] = []
|
||||
# background.add_task runs after the response in TestClient; record what would be scheduled.
|
||||
# Record the run the route submits, and what its work would call, instead of running it.
|
||||
monkeypatch.setattr(
|
||||
channels_route.scheduler,
|
||||
"run_ad_hoc",
|
||||
lambda job, fn, **kw: submitted.append((job, kw)) or fn(),
|
||||
)
|
||||
monkeypatch.setattr(channels_route, "backfill_channel_now", lambda *a, **k: calls.append(a))
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: user)
|
||||
resp = TestClient(app).patch("/api/channels/UC_wire", json={"deep_requested": True})
|
||||
return resp, calls
|
||||
return resp, submitted, calls
|
||||
|
||||
|
||||
def test_deep_request_schedules_immediate_backfill(db, monkeypatch, override_dep):
|
||||
def test_deep_request_starts_the_backfill_as_the_users_run_of_that_channel(
|
||||
db, monkeypatch, override_dep
|
||||
):
|
||||
"""It was a BackgroundTask: what it did, spent or failed with reached only the log."""
|
||||
ch, me = _setup_channel_and_sub(db, email="wire@example.com")
|
||||
resp, calls = _patch_deep_requested(db, monkeypatch, override_dep, me)
|
||||
resp, submitted, calls = _patch_deep_requested(db, monkeypatch, override_dep, me)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["deep_requested"] is True
|
||||
# scheduled exactly once, with (channel_id, user_id) — the args backfill_channel_now needs.
|
||||
assert submitted == [("channel_backfill", {"actor_id": me.id, "target": ch.id})]
|
||||
# its work runs exactly once, with (channel_id, user_id) — the args backfill_channel_now needs.
|
||||
assert calls == [(ch.id, me.id)]
|
||||
|
||||
|
||||
def test_turning_full_history_on_for_a_finished_channel_records_no_run(
|
||||
db, monkeypatch, override_dep
|
||||
):
|
||||
"""Its whole history is in already: a run would only say "nothing to do", and switching the flag
|
||||
off and on again would otherwise write one such row each time, kept for months."""
|
||||
ch, me = _setup_channel_and_sub(db, email="done@example.com")
|
||||
ch.backfill_done = True
|
||||
db.commit()
|
||||
resp, submitted, calls = _patch_deep_requested(db, monkeypatch, override_dep, me)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert (submitted, calls) == ([], [])
|
||||
|
||||
|
||||
class TestImmediateBackfillGuard:
|
||||
"""backfill_channel_now coordinates with the paced scheduler through _backfill_in_progress (per-
|
||||
channel mutual exclusion, both directions) + _immediate_running (a cap on interactive runs so a
|
||||
@@ -148,15 +171,49 @@ class TestImmediateBackfillGuard:
|
||||
def test_a_second_call_for_an_in_flight_channel_is_a_noop(self, monkeypatch):
|
||||
runner._backfill_in_progress.add("UC_busy")
|
||||
monkeypatch.setattr(runner, "SessionLocal", self._boom_session)
|
||||
runner.backfill_channel_now("UC_busy", 1) # returns immediately; _boom_session never runs
|
||||
# returns at once — _boom_session never runs — and the run says why it did nothing.
|
||||
assert runner.backfill_channel_now("UC_busy", 1) == {"skipped": "channel busy"}
|
||||
assert runner._backfill_in_progress == {"UC_busy"} # the in-flight entry is untouched
|
||||
|
||||
def test_over_the_concurrency_cap_falls_back_to_the_scheduler(self, monkeypatch):
|
||||
runner._immediate_running = runner._MAX_CONCURRENT_IMMEDIATE
|
||||
monkeypatch.setattr(runner, "SessionLocal", self._boom_session)
|
||||
runner.backfill_channel_now("UC_new", 1) # over the cap → no-op, scheduler will handle it
|
||||
# over the cap → no-op, the scheduler will handle it
|
||||
assert runner.backfill_channel_now("UC_new", 1) == {"skipped": "at capacity"}
|
||||
assert "UC_new" not in runner._backfill_in_progress
|
||||
|
||||
@staticmethod
|
||||
def _one_channel(db, monkeypatch, cid):
|
||||
ch = Channel(id=cid, title=cid, uploads_playlist_id="UU" + cid, backfill_done=False)
|
||||
me = User(email=f"{cid}@example.com")
|
||||
db.add_all([ch, me])
|
||||
db.commit()
|
||||
monkeypatch.setattr(runner, "get_service_user", lambda _db: me)
|
||||
monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: False)
|
||||
monkeypatch.setattr(runner, "YouTubeClient", lambda *a, **k: _NullYt())
|
||||
return ch, me
|
||||
|
||||
def test_a_failure_reaches_its_run_and_still_frees_the_channel(self, db, monkeypatch):
|
||||
"""The failure was logged and swallowed, so the run that now wraps it would read "ok"."""
|
||||
ch, me = self._one_channel(db, monkeypatch, "UC_fail")
|
||||
|
||||
def boom(*_a):
|
||||
raise RuntimeError("playlistItems 500")
|
||||
|
||||
monkeypatch.setattr(runner, "backfill_channel_deep", boom)
|
||||
with pytest.raises(RuntimeError, match="playlistItems 500"):
|
||||
runner.backfill_channel_now(ch.id, me.id)
|
||||
assert runner._backfill_in_progress == set() and runner._immediate_running == 0
|
||||
|
||||
def test_a_finished_backfill_says_what_it_added(self, db, monkeypatch):
|
||||
ch, me = self._one_channel(db, monkeypatch, "UC_ok")
|
||||
monkeypatch.setattr(runner, "backfill_channel_deep", lambda *_a: 7)
|
||||
monkeypatch.setattr(runner, "reconcile_full_history", lambda _db: 1)
|
||||
assert runner.backfill_channel_now(ch.id, me.id) == {
|
||||
"videos_added": 7,
|
||||
"reconciled_full": 1,
|
||||
}
|
||||
|
||||
def test_scheduler_deep_pass_skips_an_in_flight_channel(self, db, monkeypatch):
|
||||
# A channel the scheduler's query WOULD pick (deep-requested, recent-synced, not done)…
|
||||
ch = Channel(
|
||||
|
||||
@@ -14,7 +14,8 @@ import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import Channel, ChannelTag, Tag, User
|
||||
from app.models import Channel, User
|
||||
from app.reader.models import ChannelTag, Tag
|
||||
|
||||
|
||||
def test_channel_detail_returns_only_this_users_tag_ids(db, monkeypatch, override_dep):
|
||||
|
||||
@@ -18,7 +18,8 @@ from app.downloads import errors as dlerrors
|
||||
from app.downloads import formats, pipeline, quota, runner, service
|
||||
from app.downloads import probe as probemod
|
||||
from app.downloads.edit import INPUT_GUARD
|
||||
from app.models import Channel, DownloadJob, MediaAsset, PlexItem, PlexLibrary, PlexShow, Video
|
||||
from app.models import Channel, PlexItem, PlexLibrary, PlexShow, Video
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.plex import paths as plex_paths
|
||||
|
||||
# --- the spec -----------------------------------------------------------------------------------
|
||||
|
||||
@@ -9,17 +9,8 @@ from app.auth import current_user
|
||||
from app.collation import sorted_words
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import (
|
||||
Channel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
PlexCollection,
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
Subscription,
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.models import Channel, PlexCollection, PlexItem, PlexLibrary, Subscription, User, Video
|
||||
from app.reader.models import Playlist, PlaylistItem
|
||||
from datetime import datetime, timezone
|
||||
|
||||
WORDS = ["Zorró", "Életrajzi", "Ötödik", "oroszlán", "Csillag", "Cukor"]
|
||||
|
||||
@@ -20,18 +20,15 @@ from app.plex.cards import leaf_card
|
||||
from app.models import (
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
ComfortPin,
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
PlexShow,
|
||||
PlexState,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
Subscription,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.reader.models import ComfortPin, Playlist, PlaylistItem, VideoState
|
||||
|
||||
NOW = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@@ -18,7 +18,8 @@ from cryptography.fernet import Fernet
|
||||
|
||||
from app import security
|
||||
from app.downloads import credentials, formats, service
|
||||
from app.models import DownloadCredential, DownloadJob, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadCredential, DownloadJob, MediaAsset
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -24,7 +24,8 @@ from app import security
|
||||
from app.config import settings
|
||||
from app.db import engine
|
||||
from app.downloads import credentials, formats, pipeline, service
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.worker import _process_job
|
||||
|
||||
_REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
|
||||
|
||||
@@ -18,7 +18,8 @@ from sqlalchemy import select
|
||||
from app.apitokens import FilesCaller
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import service
|
||||
from app.models import DownloadJob, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.routes.downloads import cancel_download, pause_download
|
||||
from app.routes.files import delete_file, own_job_for_update
|
||||
|
||||
@@ -88,6 +89,7 @@ def test_overlapping_requests_for_one_job_release_its_hold_once(
|
||||
# The second request waits for the first one's locks instead of deciding on the same row.
|
||||
assert not finished.wait(0.5), "the second request did not wait for the first"
|
||||
transition(first, locked)
|
||||
first.commit() # the handler's commit: the service only flushes
|
||||
other.join(10)
|
||||
assert finished.is_set()
|
||||
finally:
|
||||
|
||||
@@ -13,7 +13,8 @@ its OWN session, so every test COMMITS its setup before calling it.
|
||||
from app.config import settings
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import formats, pipeline, quota
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.worker import _claim_job
|
||||
|
||||
|
||||
@@ -179,6 +180,7 @@ def test_a_suspended_users_queued_jobs_wait(db):
|
||||
|
||||
|
||||
def test_list_downloads_is_capped(db, monkeypatch):
|
||||
from app.downloads import queries
|
||||
from app.routes import downloads as dl
|
||||
|
||||
u = User(email="heavy@example.com")
|
||||
@@ -188,7 +190,7 @@ def test_list_downloads_is_capped(db, monkeypatch):
|
||||
_job(db, u.id, queue_pos=i, status="done")
|
||||
db.commit()
|
||||
|
||||
monkeypatch.setattr(dl, "MAX_LIST", 2)
|
||||
monkeypatch.setattr(queries, "MAX_LIST", 2)
|
||||
rows = dl.list_downloads(user=u, db=db)
|
||||
assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk
|
||||
|
||||
|
||||
@@ -13,7 +13,8 @@ DB lane (`db` fixture) — skipped without Postgres.
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from app.downloads import service
|
||||
from app.models import DownloadJob, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def _user(db, email="ref@example.com"):
|
||||
|
||||
@@ -4,7 +4,8 @@ you") — the only place the title lived, so a Hungarian reader got the English.
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import DownloadJob, MediaAsset, Notification
|
||||
from app.models import Notification
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user):
|
||||
|
||||
@@ -8,7 +8,8 @@ behaviour-equivalently. DB lane (`db` fixture) — skipped without Postgres.
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.models import DownloadJob, MediaAsset, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def _seed(db, *, job_status, asset_status="ready", ref_count=1, rel_path=None):
|
||||
|
||||
@@ -11,7 +11,8 @@ from sqlalchemy import text
|
||||
|
||||
from app.downloads import service
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, User
|
||||
from app.models import User
|
||||
from app.downloads.models import DownloadJob
|
||||
|
||||
|
||||
def _running_job(db) -> DownloadJob:
|
||||
|
||||
@@ -13,7 +13,7 @@ from app.config import settings
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import pipeline, quota, service
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
from app.downloads.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
|
||||
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
|
||||
|
||||
@@ -12,15 +12,8 @@ contradicts the remote's own count is not believed either.
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import (
|
||||
Channel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
PlexCollection,
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
Video,
|
||||
)
|
||||
from app.models import Channel, PlexCollection, PlexItem, PlexLibrary, Video
|
||||
from app.reader.models import Playlist, PlaylistItem
|
||||
from app.plex import sync as plex_sync
|
||||
from app.sync import playlists as plsync
|
||||
from app.youtube.client import YouTubeError
|
||||
|
||||
@@ -20,7 +20,7 @@ import yt_dlp
|
||||
from app import logs, sysconfig, worker
|
||||
from app.downloads import pipeline
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads.models import DownloadJob, MediaAsset
|
||||
from app.plex import stream as plex_stream
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Feed cursor endpoint: a malformed ?cursor= must be a clean HTTP 400, not a 500.
|
||||
|
||||
Locks the refactor that split cursor handling across modules — feedquery.decode_cursor raises a
|
||||
plain ValueError and routes/feed.get_feed translates it to 400 — so a future regression (a dropped
|
||||
Cursor handling spans modules — feedquery.decode_cursor raises a plain ValueError, feed_page turns
|
||||
it into BadCursor, and routes/feed.get_feed translates that to 400 — so a regression (a dropped
|
||||
try/except, or a new decode_cursor caller without its own catch) turns the 400 back into a 500 and
|
||||
fails right here. DB lane (`db` fixture) — skipped without Postgres.
|
||||
"""
|
||||
|
||||
@@ -14,7 +14,8 @@ import pytest
|
||||
|
||||
from app import feedquery
|
||||
|
||||
from app.models import Channel, ChannelTag, Subscription, Tag, Video, VideoState
|
||||
from app.models import Channel, Subscription, Video
|
||||
from app.reader.models import ChannelTag, Tag, VideoState
|
||||
|
||||
|
||||
def _dt(day: int) -> datetime:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user