Compare commits
490
Commits
v0.93.2
...
c8247c0dde
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8247c0dde | ||
|
|
722c37a6d0 | ||
|
|
5ba5679239 | ||
|
|
926e5e9899 | ||
|
|
9251918e08 | ||
|
|
2b6d968833 | ||
|
|
95e6b2c3d9 | ||
|
|
80c55fbf1b | ||
|
|
011f269152 | ||
|
|
dcb244dd04 | ||
|
|
08a9f4e255 | ||
|
|
1673134c36 | ||
|
|
27427eef32 | ||
|
|
a248ca6c13 | ||
|
|
620f84dba4 | ||
|
|
2efa7d5a75 | ||
|
|
827df7494c | ||
|
|
86712b6b0e | ||
|
|
90567718b2 | ||
|
|
c31d7f9d65 | ||
|
|
db56f25387 | ||
|
|
4f58d15d3f | ||
|
|
2c66ce1cf4 | ||
|
|
ce45445d17 | ||
|
|
383d394e32 | ||
|
|
06e8291896 | ||
|
|
db6b60029f | ||
|
|
02134799cc | ||
|
|
c8bc843517 | ||
|
|
759a7536b9 | ||
|
|
b2d34e6e6c | ||
|
|
c335d54929 | ||
|
|
7188ad7d71 | ||
|
|
ebbae6794c | ||
|
|
c9baf68b16 | ||
|
|
5bb101e493 | ||
|
|
54a6bbf07f | ||
|
|
8b2260c9a8 | ||
|
|
815083bcc4 | ||
|
|
96509fe356 | ||
|
|
132fb56428 | ||
|
|
f6bce23a94 | ||
|
|
fb8fb980e5 | ||
|
|
6707550513 | ||
|
|
078194731b | ||
|
|
1c4f0042d2 | ||
|
|
d22c64f4dd | ||
|
|
85aa917b38 | ||
|
|
1a1665b8ec | ||
|
|
bf99444ded | ||
|
|
07e0990ecf | ||
|
|
e2ed2bf874 | ||
|
|
b9eb8458b3 | ||
|
|
891507dc75 | ||
|
|
e6894ac0fc | ||
|
|
663a7e8012 | ||
|
|
93192dadd8 | ||
|
|
5b48018e92 | ||
|
|
8312c6d592 | ||
|
|
e1d325b942 | ||
|
|
977ba244fa | ||
|
|
89d18b46a6 | ||
|
|
c27cfdcb9c | ||
|
|
a867145bcb | ||
|
|
4d4eea2c54 | ||
|
|
c489b60c8b | ||
|
|
b4bf71ea64 | ||
|
|
72df11f2c0 | ||
|
|
0a16bc40ae | ||
|
|
520b0afb90 | ||
|
|
e301632765 | ||
|
|
50157cc2fd | ||
|
|
f221c1962a | ||
|
|
d466ae915b | ||
|
|
14598ec320 | ||
|
|
b16cf7668c | ||
|
|
3438aeb695 | ||
|
|
c003a48244 | ||
|
|
8a2334e198 | ||
|
|
1f010328df | ||
|
|
77638f7a0b | ||
|
|
2cff4ad954 | ||
|
|
8776586ca3 | ||
|
|
92a17aba83 | ||
|
|
fc53bc03bc | ||
|
|
6df257d7df | ||
|
|
5572499b56 | ||
|
|
76738f8ea9 | ||
|
|
568bf93510 | ||
|
|
942154cf49 | ||
|
|
3eb87e7108 | ||
|
|
851b97b09d | ||
|
|
ee94d1949a | ||
|
|
4f076b5d45 | ||
|
|
ece74a42ab | ||
|
|
600d5c39da | ||
|
|
4c7f73af9d | ||
|
|
5f42ff61c8 | ||
|
|
6dd82ef5e2 | ||
|
|
75f4efde98 | ||
|
|
c0059a0c4b | ||
|
|
a29cadf03e | ||
|
|
83886494ce | ||
|
|
4eaa55385f | ||
|
|
7541b1b016 | ||
|
|
24ad17fc7d | ||
|
|
48bd038c5f | ||
|
|
0fce732497 | ||
|
|
7e94da72b9 | ||
|
|
fadb4f17a9 | ||
|
|
57a81c5643 | ||
|
|
dae0b80f6a | ||
|
|
75093458c9 | ||
|
|
5db6dc8ee6 | ||
|
|
45a9ceae25 | ||
|
|
5249c41edc | ||
|
|
c3664826fe | ||
|
|
c9b8fe15a3 | ||
|
|
1044eaf8a9 | ||
|
|
ca949d22de | ||
|
|
20178f0ec4 | ||
|
|
49a9319f84 | ||
|
|
70d56ab939 | ||
|
|
3865689c26 | ||
|
|
c32429ba69 | ||
|
|
a504550196 | ||
|
|
5095047c81 | ||
|
|
4e70f168b1 | ||
|
|
7f17f488ab | ||
|
|
d1704a5519 | ||
|
|
121ebeebc1 | ||
|
|
ee5290ba01 | ||
|
|
dd6a666ad1 | ||
|
|
392099770a | ||
|
|
2aa3ac6e85 | ||
|
|
891dc628f3 | ||
|
|
0a35f2a3b1 | ||
|
|
d25a132495 | ||
|
|
61d3a1a189 | ||
|
|
d482a5e8e2 | ||
|
|
2064373ac2 | ||
|
|
0cda5b6dd4 | ||
|
|
f3c94641a7 | ||
|
|
59b27467ab | ||
|
|
02feaa9a1c | ||
|
|
c1058f1d5a | ||
|
|
23f16b69df | ||
|
|
c813814c3c | ||
|
|
ed6facb703 | ||
|
|
d44d8d6dda | ||
|
|
b20e9a3947 | ||
|
|
6073a006b8 | ||
|
|
dba4758c59 | ||
|
|
68cfc7e25d | ||
|
|
00886d92fe | ||
|
|
e7d2eb8cf6 | ||
|
|
e886890d6e | ||
|
|
5561d9a379 | ||
|
|
2768ab5bdd | ||
|
|
8c79590165 | ||
|
|
eef3ac9c5c | ||
|
|
4ba7c966d7 | ||
|
|
513dd8175a | ||
|
|
c4298b0a73 | ||
|
|
5ca2a0e6ee | ||
|
|
5257d57196 | ||
|
|
58ad95263c | ||
|
|
96ad1ef306 | ||
|
|
d1bbe4524c | ||
|
|
eecc195b7e | ||
|
|
2f901ad643 | ||
|
|
1f357b0c21 | ||
|
|
fb962682eb | ||
|
|
4e71723a07 | ||
|
|
4aaaaa7860 | ||
|
|
c73d8be094 | ||
|
|
30545d0d88 | ||
|
|
ac702192c9 | ||
|
|
24c47d12bd | ||
|
|
ab76166878 | ||
|
|
a42f346a32 | ||
|
|
f2fe0fc0df | ||
|
|
ca707fd390 | ||
|
|
2eb20ca227 | ||
|
|
adaa7dc23c | ||
|
|
91f6a77fa3 | ||
|
|
36653886d5 | ||
|
|
7e681e112e | ||
|
|
3642397f63 | ||
|
|
6d8b7be6d8 | ||
|
|
cca3315836 | ||
|
|
7f08c510bf | ||
|
|
a499f00209 | ||
|
|
c2bf8fb7f6 | ||
|
|
11d35910fd | ||
|
|
27f48b7a8b | ||
|
|
bc856ab2af | ||
|
|
0dfb4a984b | ||
|
|
5bcd1d8d8a | ||
|
|
f49789665e | ||
|
|
0e077b84c8 | ||
|
|
ce8ccfc3c0 | ||
|
|
33dec56e45 | ||
|
|
2a699e26ba | ||
|
|
13c8c05661 | ||
|
|
cc414de952 | ||
|
|
0528b35f2d | ||
|
|
cb46774851 | ||
|
|
f1cc6ce5dc | ||
|
|
4efe799eda | ||
|
|
40b27f2982 | ||
|
|
0314c4cce2 | ||
|
|
182ec31120 | ||
|
|
12a39f5291 | ||
|
|
79704b0001 | ||
|
|
4191189b4c | ||
|
|
7dbda60815 | ||
|
|
d1df76e8e5 | ||
|
|
3f6a9efd57 | ||
|
|
64e8f1da3d | ||
|
|
a1224e9a8c | ||
|
|
da4b19a39a | ||
|
|
5a74cf86b7 | ||
|
|
e432dc2780 | ||
|
|
3741d1a46d | ||
|
|
98ec1abf25 | ||
|
|
e5b155ada9 | ||
|
|
e09d613b04 | ||
|
|
38943fa93e | ||
|
|
887b4e546c | ||
|
|
df4a79a1f1 | ||
|
|
5d10a521a8 | ||
|
|
d7fb7075b8 | ||
|
|
a486c7521a | ||
|
|
76358a0d77 | ||
|
|
dc00b55feb | ||
|
|
fb6b56f121 | ||
|
|
2e2d6f5746 | ||
|
|
2ab9ff6a42 | ||
|
|
378c4daf0f | ||
|
|
8f1208fdfe | ||
|
|
52f12e5331 | ||
|
|
c4abbe48c1 | ||
|
|
4fc307c426 | ||
|
|
328d415cfe | ||
|
|
43e4929b45 | ||
|
|
5806adda7e | ||
|
|
c73761ae39 | ||
|
|
ca9ca85b18 | ||
|
|
614d728561 | ||
|
|
783f6408f9 | ||
|
|
751e1ae5ed | ||
|
|
da554df7c7 | ||
|
|
b1ff4058ac | ||
|
|
7ed2ac4e95 | ||
|
|
cff8fe9a6a | ||
|
|
471846d784 | ||
|
|
3c16012524 | ||
|
|
cdfe7a8986 | ||
|
|
3e96211db0 | ||
|
|
e9c15baae0 | ||
|
|
6edb22988c | ||
|
|
f9c3714025 | ||
|
|
a4775e1cd0 | ||
|
|
b40440b825 | ||
|
|
c4c442bc5b | ||
|
|
c74ceb73e3 | ||
|
|
e0e28c006c | ||
|
|
f8f22fc985 | ||
|
|
90e0baa870 | ||
|
|
dfd43ae89c | ||
|
|
608fde2d55 | ||
|
|
0302c466e8 | ||
|
|
a50055577a | ||
|
|
e16cdbd16f | ||
|
|
15ec67b029 | ||
|
|
ac53a3b757 | ||
|
|
e6de18b8c0 | ||
|
|
fb2e3d8490 | ||
|
|
d928d50686 | ||
|
|
540822d2b0 | ||
|
|
b4ef07d28e | ||
|
|
e995c664c6 | ||
|
|
20010178b0 | ||
|
|
78fdcda6bb | ||
|
|
98d0ca6eeb | ||
|
|
e4c80fb7bb | ||
|
|
be156fc52b | ||
|
|
25905aa786 | ||
|
|
87d71e3762 | ||
|
|
ab146211fa | ||
|
|
ec22eefd31 | ||
|
|
2543d44860 | ||
|
|
29c0715237 | ||
|
|
6b86f99249 | ||
|
|
3563858fd1 | ||
|
|
08e62bbb16 | ||
|
|
9454717f47 | ||
|
|
6ac9ffd45e | ||
|
|
fa55e01ef0 | ||
|
|
ab963c81d2 | ||
|
|
59e6cc027d | ||
|
|
ca94a716d8 | ||
|
|
63d01babe5 | ||
|
|
705377c797 | ||
|
|
2819049133 | ||
|
|
95b94a1d72 | ||
|
|
12077fbebc | ||
|
|
519fbe2a18 | ||
|
|
a39504e617 | ||
|
|
aac79ffa53 | ||
|
|
67c9f10153 | ||
|
|
584d082b6b | ||
|
|
f3a85916ec | ||
|
|
f6298d29af | ||
|
|
632e39d458 | ||
|
|
13d95ef70e | ||
|
|
0f9ed7cdbf | ||
|
|
ffcea0cb07 | ||
|
|
a3f8314c9a | ||
|
|
d5ccb1091a | ||
|
|
528774d640 | ||
|
|
3a9ea9e330 | ||
|
|
25f2d9d0e4 | ||
|
|
5ab6778b97 | ||
|
|
26697472cf | ||
|
|
2abd4ffcb6 | ||
|
|
a9ac303d3e | ||
|
|
740602b9bf | ||
|
|
e31a588f19 | ||
|
|
cda626be10 | ||
|
|
b7ce1d467c | ||
|
|
52fb999af6 | ||
|
|
e0453fa07d | ||
|
|
2525a12264 | ||
|
|
8e09dd85f8 | ||
|
|
addb3d1b32 | ||
|
|
86884e368e | ||
|
|
51f881681b | ||
|
|
27483ae1d6 | ||
|
|
595c29e17c | ||
|
|
acf93a1f36 | ||
|
|
84278293df | ||
|
|
dc24d80ed4 | ||
|
|
3810af66da | ||
|
|
eb339195c7 | ||
|
|
5d1065c1bc | ||
|
|
6ffb5bd8c1 | ||
|
|
dddf79ec2f | ||
|
|
5ebb4cb506 | ||
|
|
0b1d905866 | ||
|
|
52c15adaf2 | ||
|
|
3372626e34 | ||
|
|
98bb841a6a | ||
|
|
98bf0a9d66 | ||
|
|
17bb14fb18 | ||
|
|
70f98b352f | ||
|
|
efa4aeeeda | ||
|
|
a7d41eb5f3 | ||
|
|
76eead976f | ||
|
|
922d589b09 | ||
|
|
a975ab627a | ||
|
|
9bedd96774 | ||
|
|
383c85231a | ||
|
|
de62035889 | ||
|
|
b456232d00 | ||
|
|
c9e95ecea6 | ||
|
|
7987bd8a37 | ||
|
|
3411265dc1 | ||
|
|
098b8dbe43 | ||
|
|
f48f16830e | ||
|
|
955c23d321 | ||
|
|
d3e1d8219c | ||
|
|
4480839db0 | ||
|
|
8e25567ff2 | ||
|
|
8f65bd184e | ||
|
|
ae9c248987 | ||
|
|
bd6aaa1e43 | ||
|
|
2c65edfffd | ||
|
|
6c2c01e67b | ||
|
|
ef343dbec0 | ||
|
|
a58a398427 | ||
|
|
d510c8a534 | ||
|
|
dc5206f9d3 | ||
|
|
89e65ca331 | ||
|
|
17916b411a | ||
|
|
3ef2c38eb4 | ||
|
|
4c63d6660d | ||
|
|
a453b77f04 | ||
|
|
8e1004fe6f | ||
|
|
663f30feeb | ||
|
|
6653d81a80 | ||
|
|
21df7c8348 | ||
|
|
97381e65d0 | ||
|
|
f8ced6d819 | ||
|
|
f254588409 | ||
|
|
3e7c79fa69 | ||
|
|
47c12f9405 | ||
|
|
bd0914b7ed | ||
|
|
079cee4e9c | ||
|
|
edf2f22ef9 | ||
|
|
e1ef56cdbc | ||
|
|
4d0abc61d5 | ||
|
|
7c2b22bafa | ||
|
|
5e5c8a42a9 | ||
|
|
2b3094ffc5 | ||
|
|
9c91eb5d22 | ||
|
|
2df60fbf07 | ||
|
|
a2d17d693c | ||
|
|
c4917619ee | ||
|
|
760bdae278 | ||
|
|
1b946113c6 | ||
|
|
b0e51c6f89 | ||
|
|
7455fff401 | ||
|
|
aaf1843b09 | ||
|
|
a4737b92e4 | ||
|
|
75ae810866 | ||
|
|
9d7526ee57 | ||
|
|
fa0b0edd63 | ||
|
|
bfc6d199c0 | ||
|
|
4d6e59b80a | ||
|
|
4646aae320 | ||
|
|
eddeca82d3 | ||
|
|
69edeecc45 | ||
|
|
6088e476a7 | ||
|
|
ce171fa954 | ||
|
|
113da76c36 | ||
|
|
4ef11901db | ||
|
|
29abf609f3 | ||
|
|
e13d93a3ed | ||
|
|
b9850b1710 | ||
|
|
905a9f2856 | ||
|
|
1d8b83c37c | ||
|
|
62dbf35ea3 | ||
|
|
e9ccd9bc98 | ||
|
|
cc8e0648db | ||
|
|
ff80cc96fd | ||
|
|
295c1db5f1 | ||
|
|
0521149cf1 | ||
|
|
b4a6569d0d | ||
|
|
1833f594cb | ||
|
|
1615838f33 | ||
|
|
7fa8f4ebde | ||
|
|
6354454cc9 | ||
|
|
c47c193ad1 | ||
|
|
db10bfbbca | ||
|
|
dac9d89f91 | ||
|
|
81936e9f56 | ||
|
|
86f467bd65 | ||
|
|
7d480e2ad0 | ||
|
|
ad136c3fe8 | ||
|
|
9c4a999e54 | ||
|
|
3f5634a44f | ||
|
|
f4bc60d659 | ||
|
|
7055548260 | ||
|
|
477271907e | ||
|
|
50923bb684 | ||
|
|
d2e40dcffb | ||
|
|
58878958d8 | ||
|
|
b47d22cf3b | ||
|
|
e804ff3b8e | ||
|
|
cd6f119883 | ||
|
|
321776480e | ||
|
|
7c0647f87a | ||
|
|
efd7b217ca | ||
|
|
3d7ef9e945 | ||
|
|
7fce7a7ce5 | ||
|
|
f5018bdcbc | ||
|
|
b82c22b888 | ||
|
|
f839a461e6 | ||
|
|
8525666913 | ||
|
|
3f24248e01 | ||
|
|
6b86673af4 | ||
|
|
d924c665f8 | ||
|
|
aa51644f5f | ||
|
|
378b7559e4 | ||
|
|
05121fbc99 | ||
|
|
bf518223e5 | ||
|
|
332d24a646 | ||
|
|
c5e022f056 | ||
|
|
4065ac7e6a | ||
|
|
ce08fc2aec | ||
|
|
ea6d829c29 | ||
|
|
6a7ff330f6 | ||
|
|
7e1aeeb9fe | ||
|
|
7d682c90d5 | ||
|
|
cc9fc3b544 | ||
|
|
c9101e3450 | ||
|
|
51daf3e388 | ||
|
|
09d351a0e7 |
@@ -4,6 +4,7 @@
|
||||
*.env.local
|
||||
**/__pycache__/
|
||||
**/*.pyc
|
||||
**/.coverage
|
||||
.venv/
|
||||
venv/
|
||||
# Docker patterns are anchored at the context root with NO implicit `**/` (unlike .gitignore, where
|
||||
|
||||
+6
-6
@@ -28,7 +28,7 @@ ALLOWED_EMAILS=
|
||||
# Admin emails (subset of the above) get the admin role.
|
||||
ADMIN_EMAILS=
|
||||
|
||||
# Logging (E22). LOG_LEVEL is the BOOTSTRAP level — the level a process starts at and the only one
|
||||
# Logging. LOG_LEVEL is the BOOTSTRAP level — the level a process starts at and the only one
|
||||
# the worker/download child read; the API's running level is the admin Configuration page's
|
||||
# `log_level` knob, which takes effect without a restart. TRACE | DEBUG | INFO | WARNING | ERROR, in
|
||||
# any letter case; any other value stops the boot with an error. LOG_FORMAT is "json" (one object per
|
||||
@@ -73,13 +73,13 @@ SMTP_FROM=
|
||||
# of them and false on the rest, to avoid double quota use and write races.
|
||||
SCHEDULER_ENABLED=true
|
||||
|
||||
# --- Download center ---
|
||||
# The Download Center adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small
|
||||
# --- Files library & downloads ---
|
||||
# Downloading adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small
|
||||
# `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose.
|
||||
# Downloaded media defaults to a Docker-managed named volume. Set DOWNLOAD_HOST_PATH to a host
|
||||
# The Files library defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host
|
||||
# directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there.
|
||||
# The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000 <dir>.
|
||||
# DOWNLOAD_HOST_PATH=/mnt/media/youtube
|
||||
# FILES_HOST_PATH=/mnt/media/youtube
|
||||
|
||||
# --- Plex integration (optional) ---
|
||||
# The optional Plex module browses/plays your Plex library, playing the LOCAL physical file
|
||||
@@ -89,6 +89,6 @@ SCHEDULER_ENABLED=true
|
||||
# to /plex-media inside the container (the local side of the admin "path map", whose default is
|
||||
# /data=/plex-media). Leave unset if you don't use Plex.
|
||||
# PLEX_MEDIA_HOST_PATH=/mnt/media
|
||||
# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to DOWNLOAD_ROOT; point it at
|
||||
# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to FILES_ROOT; point it at
|
||||
# fast local storage if your download volume is slow.
|
||||
# PLEX_HLS_DIR=/var/tmp/plex-hls
|
||||
|
||||
@@ -2,8 +2,11 @@
|
||||
# whole-repo reformatting does not bury the real authorship of each line.
|
||||
# Configure once per clone: git config blame.ignoreRevsFile .git-blame-ignore-revs
|
||||
#
|
||||
# Prettier one-time repo-wide format pass (R2 S1b, 2026-07-21)
|
||||
# Prettier one-time repo-wide format pass (2026-07-21)
|
||||
51c36c2071728d2cd7ae580b7bfdf21f6867a347
|
||||
|
||||
# ruff format one-time backend + scripts pass (line-length 100, 2026-08-23)
|
||||
b1832afd180866b922521148603db447020c57a4
|
||||
|
||||
# Plan-code purge from comments, one-time (2026-10-04)
|
||||
4aaaaa786017278460e0b4f94cc0d3374e0e37e2
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
||||
# independent of any one machine. Two parallel jobs match the two local lanes:
|
||||
# frontend — tsc (3 projects) + eslint + prettier --check + knip + vitest
|
||||
# backend — ruff + the full pytest suite (pure-logic + the DB-backed lane) against a real Postgres
|
||||
#
|
||||
# Runner: the lab act_runner (LXC 115) mounts the Docker socket, so service containers work. The backend
|
||||
# runs pytest NATIVELY against a `postgres` service (not the local docker-compose.test.yml, whose
|
||||
# bind-mount is unreliable under a mounted-socket runner); fidelity is kept by installing the SAME pinned
|
||||
# requirements, Postgres 16, and ffmpeg, and by setting SIFTLODE_TEST_REQUIRE_DB=1 so a DB/ffmpeg that
|
||||
# failed to come up is a hard error, never a silent skip.
|
||||
#
|
||||
# The frontend container's node major is NOT free: it must equal the prod build image's and the dev
|
||||
# machine's, because vitest/tsc read the engine's own Intl and web-API implementations and those split
|
||||
# between majors. `.nvmrc` is the single source of truth and
|
||||
# `frontend/src/lib/nodeEngine.contract.test.ts` fails this gate if any of them drift apart.
|
||||
name: gate
|
||||
|
||||
# Branch pushes only. A release pushes `main` and then the `v<ver>` tag, and a bare `on: [push]`
|
||||
# treated the tag as a second push of the IDENTICAL commit — every release ran this whole gate
|
||||
# twice, to the same verdict, for nothing (runs #81-#90 are five such pairs). `branches: ['**']`
|
||||
# keeps every branch covered and drops tag refs.
|
||||
#
|
||||
# It has to be `branches`, NOT the `tags-ignore` this obviously wants to be. Gitea's matchPushEvent
|
||||
# requires EVERY named filter to match, and each one bails on the wrong ref kind: `tags-ignore`
|
||||
# starts with `if !refName.IsTag() { break }`, so a filter meant to skip tags rejects every BRANCH
|
||||
# push and the gate stops running entirely. `branches` bails the same way on a tag, which is the
|
||||
# direction we want. (Measured in go-gitea v1.27.3 modules/actions/workflows.go, not assumed — it
|
||||
# differs from GitHub, where `tags-ignore` alone leaves branches unfiltered.)
|
||||
#
|
||||
# If that filter were ever wrong the gate would not run at all — which is why `siftlode release`
|
||||
# now ends by reading the verdict back for the sha it pushed, and says so out loud when there is
|
||||
# no run for it. A gate that silently stops running is the failure this repo already lived through.
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
container: node:24-bookworm
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: npm ci
|
||||
- run: npm run typecheck
|
||||
- run: npm run lint
|
||||
- run: npm run format:check
|
||||
- run: npm run knip
|
||||
- run: npm run test
|
||||
|
||||
backend:
|
||||
runs-on: ubuntu-latest
|
||||
# A combined python+node image: python 3.12 (matches the prod Dockerfile) for the tests, and node
|
||||
# so the JS actions (checkout) run. actions/setup-python is NOT usable here — the runner can't reach
|
||||
# GitHub's python-versions download, so it must be baked into the image.
|
||||
#
|
||||
# This node runs the checkout action, never our code, so it is deliberately OUTSIDE the engine
|
||||
# contract the frontend job is held to — it tracks `.nvmrc` as hygiene (node 20 went EOL in April
|
||||
# 2026), but a missing upstream tag here must never be able to block the gate.
|
||||
container: nikolaik/python-nodejs:python3.12-nodejs24
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: siftlode
|
||||
POSTGRES_PASSWORD: siftlode
|
||||
POSTGRES_DB: siftlode_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U siftlode -d siftlode_test"
|
||||
--health-interval 2s
|
||||
--health-timeout 3s
|
||||
--health-retries 30
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg://siftlode:siftlode@postgres:5432/siftlode_test
|
||||
SIFTLODE_TEST_REQUIRE_DB: "1"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# ffmpeg for the poster/thumbnail tests; postgresql-client for the readiness wait below.
|
||||
- run: apt-get update && apt-get install -y --no-install-recommends ffmpeg postgresql-client
|
||||
- name: Wait for Postgres
|
||||
run: until pg_isready -h postgres -U siftlode -d siftlode_test; do sleep 1; done
|
||||
- run: pip install --no-cache-dir -r backend/requirements.lock -r backend/requirements-dev.txt
|
||||
- name: ruff
|
||||
run: python -m ruff check --config backend/ruff.toml backend scripts
|
||||
- name: ruff format --check
|
||||
run: python -m ruff format --check --config backend/ruff.toml backend scripts
|
||||
- name: pytest
|
||||
run: python -m pytest
|
||||
working-directory: backend
|
||||
# Dependency CVE audit over the pinned lock. Non-blocking for now (informational) — a real CVE
|
||||
# shows up in the run without failing the gate; tighten to a hard fail once the noise is known.
|
||||
- name: pip-audit (advisory)
|
||||
run: pip install --no-cache-dir pip-audit && pip-audit -r backend/requirements.lock
|
||||
continue-on-error: true
|
||||
+8
-2
@@ -9,6 +9,7 @@ __pycache__/
|
||||
venv/
|
||||
.mypy_cache/
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
.ruff_cache/
|
||||
|
||||
# Node / frontend
|
||||
@@ -28,12 +29,17 @@ frontend/e2e/.auth/
|
||||
*.sql.gz
|
||||
backups/
|
||||
|
||||
# Download center: locally downloaded media (the DOWNLOAD_ROOT bind mount at the repo root).
|
||||
# Download center: locally downloaded media (the FILES_ROOT bind mount at the repo root).
|
||||
# Anchored with a leading slash so it does NOT also ignore the backend/app/downloads package.
|
||||
/downloads/
|
||||
/files/
|
||||
|
||||
# OS / editor
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# TypeScript incremental build artifacts — machine-local, absolute-path-bearing, and no gate lane
|
||||
# reads or regenerates them, so a committed one is stale from the commit that adds it.
|
||||
*.tsbuildinfo
|
||||
frontend/e2e/.speed/
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
3.12
|
||||
@@ -0,0 +1,15 @@
|
||||
# The CVE gate's allowlist. The publish gate scans the built image and the committed tree with
|
||||
# Trivy and fails on any HIGH or CRITICAL finding that has a fix available. A finding that cannot be
|
||||
# fixed yet is let through ONLY by an entry here, and every entry carries both:
|
||||
# statement — why it is acceptable for now (what is exposed, what is not)
|
||||
# expired_at — the date it stops being accepted; after it, the gate fails again on its own
|
||||
# A contract test (frontend/src/lib/gates.contract.test.ts) refuses an entry without either, and
|
||||
# docs/gates.md lists them.
|
||||
#
|
||||
# Example:
|
||||
# vulnerabilities:
|
||||
# - id: CVE-2026-00000
|
||||
# statement: in libfoo's XML path, which nothing in the image calls; no fixed Debian build yet
|
||||
# expired_at: 2026-12-31
|
||||
vulnerabilities: []
|
||||
secrets: []
|
||||
+991
-1374
File diff suppressed because it is too large
Load Diff
@@ -5,19 +5,23 @@ be developed that way. This file is the rules of the road: read it before you to
|
||||
loaded into every agent session, so keep it **short and true**. A rule here that has gone stale is
|
||||
worse than no rule — if you change how the code works, change this file in the same commit.
|
||||
|
||||
Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a download center). It is a
|
||||
Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a files library with downloads). It is a
|
||||
**modular monolith**: one Postgres, one FastAPI process (uvicorn + APScheduler), one worker process
|
||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 18 + react-query frontend with a router-free navigation
|
||||
provider.
|
||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 19 (with the React Compiler) + react-query frontend, with a
|
||||
router-free navigation provider.
|
||||
|
||||
## The gate — never merge or ship red
|
||||
|
||||
The local gate is **ruff (`check` + `format --check`) + typecheck (3 tsc projects) + eslint + prettier
|
||||
--check + knip + vitest + pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
||||
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
||||
The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + the lock check + typecheck (3 tsc projects) +
|
||||
eslint + prettier --check + knip + vitest + vite build + pytest + pip-audit + npm audit**. It runs
|
||||
before every `release` merges and on every `publish`, which **refuses to build on failure — there is deliberately no `--skip`
|
||||
flag**; a release also requires the Playwright suite, and the image must pass a Trivy scan before it is pushed. Run it and get it
|
||||
green **before** any merge, release, or handing work to review.
|
||||
|
||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` + `ruff format --check backend` and `pytest` (the DB-lane needs Postgres).
|
||||
- Full gate: `bash scripts/check.sh` (`lint` for the static lanes only); the private ops dispatcher's `siftlode check` runs the same script. Single lanes: [`docs/development.md`](docs/development.md) § The gate.
|
||||
- **There is no CI** (owner, 2026-10-04): this gate, run here, is the only one — green before every hand-over, and `release` refuses to merge without it.
|
||||
- **A gate with any real hit is never switched off or narrowed** — not a lint rule, a compiler flag, a lane or a release step. Fix the sites, or suppress each one inline naming its invariant (`-- why`, lint-enforced). What stays narrowed is in [`docs/gates.md`](docs/gates.md), and a contract test fails when a narrowing is missing from it. The effective ESLint config of every scope and the gate/lane lines are pinned in `frontend/src/lib/gates.snapshot.json`: any gate change fails until you regenerate it (`GATE_SNAPSHOT=update`, see gates.md) — then read that diff as a gate change. A comment saying "X is enforced by Y" is checked by running Y (E32: one such comment hid 95 functions the React Compiler skipped).
|
||||
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` (no burn-down mandate, but none was ever triaged: a `reportOptional*` entry in code you touch may be a real bug). A run that fixed some rewrites it — commit that.
|
||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||
|
||||
## Workflow
|
||||
@@ -45,21 +49,24 @@ how it gets built. Concretely:
|
||||
behind each option and the weight of the decision. These decision points are where the product owner
|
||||
engages with the build and learns from it, so make them understandable — a reasoned menu, not a bare one.
|
||||
- **Mockups for design/UX decisions.** For any visual or interaction choice, show mockup options to
|
||||
choose from before building.
|
||||
choose from before building. How the UI looks and behaves is decided in
|
||||
[`docs/DESIGN.md`](docs/DESIGN.md) (its variants render in the dev-only `/design-lab`): build to it,
|
||||
and change it only through its decision log.
|
||||
- **Challenge unsound requests.** If a request is technically ill-advised and you can back it with real
|
||||
arguments, push back or propose an alternative — don't comply blindly.
|
||||
- **Verify UI in the browser before calling it done.** tsc/unit/DOM tests miss visual and interaction
|
||||
bugs; do a real visual check of any UI change.
|
||||
bugs; do a real visual check of any UI change, and run DESIGN.md's four-step Conformance check.
|
||||
- **Be terse everywhere else.** Outside those decision points, lead with the answer or decision. During
|
||||
implementation and debugging especially: no restating the same point reworded, no filler.
|
||||
|
||||
## Architecture shape (don't fight it)
|
||||
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`,
|
||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell (a shared `/watch` link boots from its own
|
||||
entry, `watch.html` → `watchMain.tsx`, held app-free by a contract test) → `moduleRegistry` route-chunks → module
|
||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||
its menus, `Modal`/`Overlay`, `Button`/`form`, `DataTable`/`VirtualGrid`, `QueryState`, …). That
|
||||
boundary is one-way and **lint-enforced**, not just described: a platform component may not import a
|
||||
@@ -74,29 +81,40 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
|
||||
## Canonical patterns — when you have a choice, pick THIS
|
||||
|
||||
| Area | Use | Not |
|
||||
| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------ |
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE action button | `ui/Button` — pick a `SIZES` rung | a hand-rolled `<button className="px-3 py-2 …">` |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
| Area | Use | Not |
|
||||
| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Request SIZE | nothing — `bodylimit` caps every route (SEC-20); need more? add a ceiling to `ROUTE_MAX_BODY_BYTES` | `capped_body` on a route that does not read raw bytes itself |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE player on a native media element | `lib/mediaSession` (`useMediaSession`) + `lib/mediaKeys` + `lib/resume` (`useResume`), the element held as STATE | a private `navigator.mediaSession` / `keydown` / position-saving effect per player |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
||||
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
||||
| FE effect that must not re-subscribe | `useEffectEvent` (React 19.2+); a handler or timer: `useLatest` / `useStableCallback` | a render-phase `ref.current = x` — the React Compiler skips the component |
|
||||
| FE hook suppression | an inline `eslint-disable-next-line` NAMING the invariant | `eslint-suppressions.json` (deliberately absent — E27 S2) |
|
||||
| FE popover / scroll-fade ref | destructure it: `const { setReference } = pop.refs`, then `ref={setReference}` | `ref={pop.refs.setReference}` / `ref={fade.ref}` — the compiler then treats the whole object as a ref and skips |
|
||||
| FE async handler with cleanup | `try { … } catch { … }`, then the cleanup after it (steps that `return` early go in their own function); a conditional, loop or `?.` the `try` needs goes before it or into a module-level helper | `try/finally`, or a `?:` / `??` / loop / `?.` inside `try` — the React Compiler skips the whole component (and `react-hooks/todo` misses most of these) |
|
||||
|
||||
Not repeated here, because a lint rule refuses the wrong one and its message names the right one:
|
||||
buttons (`ui/Button`, `IconButton`, `Pressable`, `Chip`, `DragHandle`), text fields and native
|
||||
controls (`ui/Input`, `Checkbox`, `Slider`, …), glass surfaces (`SURFACE.*`), a native `<select>`
|
||||
(`SegmentedToggle` / `FilterSelect`), a picture (`ui/RemoteImage`, never a bare `<img>`), and network calls (`req` / `uploadReq` / `beacon`, `lib/http.ts`).
|
||||
|
||||
## House rules that bite (learned from real bugs)
|
||||
|
||||
@@ -109,7 +127,8 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
"no results". Reads degrade honestly; writes are conditional (`UPDATE … WHERE status='running'`).
|
||||
- **i18n parity.** HU/EN, zero key drift. Every user-facing string is translated.
|
||||
- **Comment the "why", keep it current.** This codebase uses comments as its architecture doc; a stale
|
||||
"why" is a liability, so update it when the behavior changes.
|
||||
"why" is a liability, so update it when the behavior changes. Say the why itself, never the plan
|
||||
step that produced it: no epic, sprint or review-round ids in the code (`planCodes.contract.test.ts`).
|
||||
|
||||
---
|
||||
|
||||
|
||||
+21
-9
@@ -11,7 +11,7 @@ ARG BUILD_DATE=
|
||||
|
||||
# Stage 1: build the frontend SPA
|
||||
# The major is pinned to `.nvmrc` and held there by frontend/src/lib/nodeEngine.contract.test.ts:
|
||||
# this image, the CI gate's image and the dev machine must be ONE engine (see that test for why).
|
||||
# this image and the dev machine the gate runs on must be ONE engine (see that test for why).
|
||||
FROM node:24-alpine AS frontend
|
||||
WORKDIR /fe
|
||||
# Deps first, keyed on the lockfile, so editing app source never re-resolves node_modules.
|
||||
@@ -39,22 +39,31 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
WORKDIR /app
|
||||
|
||||
# Create the app user BEFORE the copies so files land already-owned via COPY --chown (a trailing
|
||||
# `chown -R /app` would rewrite every file into a duplicate layer). /downloads must be appuser-owned
|
||||
# `chown -R /app` would rewrite every file into a duplicate layer). /files must be appuser-owned
|
||||
# so a fresh named volume mounted there inherits that ownership (Docker copies the image dir's
|
||||
# ownership into a new empty volume) — the worker/API can then write without a manual chown. A bind
|
||||
# mount to a host path instead needs that path writable by this uid (see docs/self-hosting.md).
|
||||
RUN adduser --disabled-password --gecos "" appuser \
|
||||
&& mkdir -p /downloads \
|
||||
&& chown appuser /app /downloads
|
||||
&& mkdir -p /files \
|
||||
&& chown appuser /app /files
|
||||
|
||||
# ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode (E6 S1).
|
||||
# ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode.
|
||||
# intel-media-va-driver (iHD): the VA driver ffmpeg's h264_vaapi needs — libva2 ships in the base but
|
||||
# no driver, so without this `vainfo`/`h264_vaapi` fail and transcode silently falls back to libx264.
|
||||
# Harmless on hosts with no /dev/dri (the driver just goes unused). Deno: the JavaScript runtime yt-dlp
|
||||
# uses to solve YouTube's "n" signature challenge for the web player clients (required alongside the PO
|
||||
# token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
||||
# `apt-get upgrade`: the base image is a snapshot, and the Debian security fixes released since it
|
||||
# was cut are only in the archive — the first Trivy scan found one (pcre2) fixed upstream but not in it.
|
||||
# APT_REFRESH: the upgrade only runs when this layer is rebuilt, and a build cache keeps it for as long
|
||||
# as nothing above it changes — a publish shipped an image whose curl and librsvg were a release behind
|
||||
# fixes already in the archive. The release build passes the day here, so the layer is rebuilt at
|
||||
# least daily; empty (the default, a self-hosted build) leaves the cache alone.
|
||||
ARG DENO_VERSION=v2.9.1
|
||||
RUN apt-get update \
|
||||
ARG APT_REFRESH=
|
||||
RUN echo "apt refresh: ${APT_REFRESH:-cached}" \
|
||||
&& apt-get update \
|
||||
&& apt-get upgrade -y --no-install-recommends \
|
||||
&& apt-get install -y --no-install-recommends ffmpeg intel-media-va-driver unzip curl ca-certificates \
|
||||
&& curl -fsSL "https://github.com/denoland/deno/releases/download/${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o /tmp/deno.zip \
|
||||
&& unzip -o /tmp/deno.zip -d /usr/local/bin \
|
||||
@@ -76,11 +85,14 @@ RUN printf 'Types: deb\nURIs: http://deb.debian.org/debian\nSuites: bookworm\nCo
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install from the PINNED lock (requirements.lock, generated from requirements.txt) so the image is
|
||||
# byte-reproducible and the exact versions are the ones pip-audit scanned in CI — not whatever floats
|
||||
# up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
||||
# byte-reproducible and the exact versions are the ones the gate's pip-audit scanned — not whatever
|
||||
# floats up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
||||
# pip itself is removed once it has installed the lock: nothing runs it at runtime, and it carries
|
||||
# its own VENDORED copies of urllib3, msgpack and setuptools, which Trivy reports as the image's own —
|
||||
# an attack surface and a CVE stream for a tool the running app never calls.
|
||||
COPY backend/requirements.lock .
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --upgrade pip && pip install -r requirements.lock
|
||||
pip install --upgrade pip && pip install -r requirements.lock && pip uninstall -y pip
|
||||
|
||||
COPY --chown=appuser backend/ .
|
||||
# Read at runtime by config.py as the app_version (a committed single source of truth).
|
||||
|
||||
@@ -22,9 +22,11 @@ blocker and SponsorBlock keep working.
|
||||
tags to slice the feed by.
|
||||
- **Playlists with two-way YouTube sync** — build them locally, keep them in sync in both directions.
|
||||
- **In-app player** with resume, plus keyboard/scroll controls.
|
||||
- **Download Center** — save videos to the server with yt-dlp in a Plex-friendly layout (format
|
||||
presets, per-user storage quota), trim / crop / split & join them in a built-in editor, then save
|
||||
to your device, share with another user, or hand out a public watch link.
|
||||
- **Files** — one library for everything you keep: paste a link anywhere in the app to download it
|
||||
with yt-dlp (format presets, per-user storage quota, the running downloads in a tray), upload your
|
||||
own files, sort them into folders, trim / crop / split & join videos in a built-in editor, play them
|
||||
where you left off, then save to your device, share with another user, or hand out a public watch
|
||||
link that plays on any phone.
|
||||
- **Plex library** *(optional)* — point Siftlode at your own Plex server to browse and play its
|
||||
movies and shows alongside your feed, with watch state kept in sync.
|
||||
- **Multi-user** with per-user private state, a shared catalog, and a fair daily API-quota guard.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,7 +6,7 @@ Create Date: 2026-07-07
|
||||
|
||||
Some sources (a direct media URL like a reddit v.redd.it HLS manifest) carry no thumbnail, so the
|
||||
download shows a blank image box. The worker now extracts a representative frame with ffmpeg and
|
||||
saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
|
||||
saves it as a poster sidecar; `poster_path` (relative to files_root) records it so the card /
|
||||
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
|
||||
"""
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""add tagline_hu to plex_items + plex_shows — the HU overlay's tagline (E30 S1)
|
||||
|
||||
Revision ID: 0091_plex_tagline_hu
|
||||
Revises: 0090_debug_session_draft
|
||||
Create Date: 2026-09-29
|
||||
|
||||
The Hungarian overlay section lists a tagline beside the title and summary; the info page showed the
|
||||
English one to a Hungarian reader. NULL until the next overlay sync fills it (serving falls back to EN).
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0091_plex_tagline_hu"
|
||||
down_revision: str | None = "0090_debug_session_draft"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("plex_items", sa.Column("tagline_hu", sa.Text(), nullable=True))
|
||||
op.add_column("plex_shows", sa.Column("tagline_hu", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("plex_shows", "tagline_hu")
|
||||
op.drop_column("plex_items", "tagline_hu")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add biography_hu to plex_people — TMDB's Hungarian biography (E30 S4)
|
||||
|
||||
Revision ID: 0092_plex_person_biography_hu
|
||||
Revises: 0091_plex_tagline_hu
|
||||
Create Date: 2026-09-29
|
||||
|
||||
The person page showed a Hungarian reader the English biography although TMDB has a Hungarian one for
|
||||
most people. The person fetch now appends `translations` and keeps the Hungarian biography beside the
|
||||
English one. Already-enriched people are un-stamped, so their next page open refetches them once (the
|
||||
same lazy path as a first open) and fills it; until then the English biography stands in.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0092_plex_person_biography_hu"
|
||||
down_revision: str | None = "0091_plex_tagline_hu"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("plex_people", sa.Column("biography_hu", sa.Text(), nullable=True))
|
||||
op.execute(
|
||||
"UPDATE plex_people SET tmdb_enriched_at = NULL "
|
||||
"WHERE tmdb_person_id IS NOT NULL AND tmdb_enriched_at IS NOT NULL"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("plex_people", "biography_hu")
|
||||
@@ -0,0 +1,29 @@
|
||||
"""re-fetch linked people — "Known for" ranked by votes, without guest visits (E30 S6)
|
||||
|
||||
Revision ID: 0093_plex_person_known_for_votes
|
||||
Revises: 0092_plex_person_biography_hu
|
||||
Create Date: 2026-09-30
|
||||
|
||||
A person's "Known for" pool was their credits ranked by TMDB popularity, so talk-show visits as
|
||||
themselves crowded out the films (Tom Cruise: 14 of 15 were talk shows, none in the library). The pool is
|
||||
now ranked by vote count, without guest visits, with directed works, and deeper. Already-enriched people
|
||||
are un-stamped so the people_enrich job fetches them again; their biography stays on the row meanwhile.
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0093_plex_person_known_for_votes"
|
||||
down_revision: str | None = "0092_plex_person_biography_hu"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"UPDATE plex_people SET tmdb_enriched_at = NULL "
|
||||
"WHERE tmdb_person_id IS NOT NULL AND tmdb_enriched_at IS NOT NULL"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass # the watermark only says "fetched"; a refetch under the old ranking is not worth undoing to
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Hungarian genres on every Plex row, not only on overlaid ones (E30 review)
|
||||
|
||||
Revision ID: 0094_plex_genres_hu_everywhere
|
||||
Revises: 0093_plex_person_known_for_votes
|
||||
Create Date: 2026-09-30
|
||||
|
||||
The HU genre facets and filter read `genres_hu`, which only the overlay pass wrote — so a library with
|
||||
no HU twin, or a guid-less row, offered English genres a Hungarian filter could never match. The sync
|
||||
now writes `genres_hu` for every row (plex/sync.py `_apply_facet_fields`); this fills the rows it has
|
||||
not reached yet, from their own English list through the same map.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from app.plex.genres import hu_genres
|
||||
|
||||
revision: str = "0094_plex_genres_hu_everywhere"
|
||||
down_revision: str | None = "0093_plex_person_known_for_votes"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
for table in ("plex_items", "plex_shows"):
|
||||
rows = conn.execute(
|
||||
sa.text(
|
||||
f"SELECT id, genres FROM {table} "
|
||||
"WHERE genres_hu IS NULL AND jsonb_typeof(genres) = 'array'"
|
||||
)
|
||||
).fetchall()
|
||||
updates = [
|
||||
{"id": rid, "g": json.dumps(hu)} for rid, genres in rows if (hu := hu_genres(genres))
|
||||
]
|
||||
if updates:
|
||||
conn.execute(
|
||||
sa.text(f"UPDATE {table} SET genres_hu = CAST(:g AS jsonb) WHERE id = :id"), updates
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass # derived from `genres`; the sync writes the same values again
|
||||
@@ -0,0 +1,83 @@
|
||||
"""videos: the per-channel index covers what the channel manager and the feed sidebar read (E31 S4d)
|
||||
|
||||
The channel manager (`GET /api/channels`) sums each subscribed channel's videos: count, last upload, total
|
||||
duration. The feed sidebar (`/feed/summary`, `/feed/years`) counts the reader's videos per channel and per
|
||||
year under the default filters: available, not via explore, not a short, not live. Every one of them found
|
||||
its rows through `ix_videos_channel_pub_desc` (channel_id, published_at DESC NULLS LAST, id) and then read
|
||||
each video's heap page for the rest: ~270k rows at ~2.5 KB each, 681 MB on prod. That stays in cache only
|
||||
while nothing else needs the memory. Measured on prod (2026-10-02): `/api/channels` 51–61 ms warm, but
|
||||
2.25 s and 1.96 s on a first open in the browser and 6.4 s under EXPLAIN with 30k blocks read from disk;
|
||||
`/feed/summary` and `/feed/years` ~140 ms warm, ~800 ms cold.
|
||||
|
||||
The same key with those columns INCLUDEd makes them Index Only Scans. On a prod-copy localdev, after a
|
||||
VACUUM, the manager's sums read 0 heap pages and the summary reads the 25 MB index instead of the heap.
|
||||
The visibility map is what lets them skip the heap; prod's autovacuum kept it complete (relallvisible ==
|
||||
relpages). The key columns are unchanged, so every reader of the old index (the per-channel feed, the FK
|
||||
cascade, the sync upsert, channel detail) rides this one. The old index is dropped.
|
||||
tests/test_channel_list_index_only.py holds the queries and this column list together.
|
||||
|
||||
CONCURRENTLY (autocommit_block) builds it without blocking a live prod. IF (NOT) EXISTS keeps the drift
|
||||
test's downgrade → re-upgrade round trip idempotent. A re-run after a build that failed part-way rebuilds
|
||||
the INVALID leftover, and the old index is dropped only once the new one is valid: a re-run that kept the
|
||||
leftover and dropped the old one left `videos` with no usable channel_id index (E31 S5 review).
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
revision: str = "0095_videos_channel_cover_index"
|
||||
down_revision: str | None = "0094_plex_genres_hu_everywhere"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def _valid(name: str) -> bool | None:
|
||||
"""The index's `indisvalid`, or None when there is no index of that name."""
|
||||
row = (
|
||||
op.get_bind()
|
||||
.execute(
|
||||
text(
|
||||
"SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid "
|
||||
"WHERE c.relname = :name"
|
||||
),
|
||||
{"name": name},
|
||||
)
|
||||
.first()
|
||||
)
|
||||
return None if row is None else bool(row[0])
|
||||
|
||||
|
||||
def _build(name: str, ddl: str) -> None:
|
||||
"""CREATE INDEX CONCURRENTLY `name` (`ddl` is what follows the name), re-runnable. A build that
|
||||
failed part-way leaves an INVALID index under the name, and `IF NOT EXISTS` would keep it: the
|
||||
planner never uses it, and nothing says so (E31 S5 review). Such a leftover is dropped and built
|
||||
again."""
|
||||
if _valid(name) is False:
|
||||
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||
op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} {ddl}")
|
||||
|
||||
|
||||
def _swap(build: str, ddl: str, drop: str) -> None:
|
||||
_build(build, ddl)
|
||||
if not _valid(build):
|
||||
raise RuntimeError(f"{build} did not build valid; {drop} is kept")
|
||||
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {drop}")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.get_context().autocommit_block():
|
||||
_swap(
|
||||
"ix_videos_channel_pub_cover",
|
||||
"ON videos (channel_id, published_at DESC NULLS LAST, id) "
|
||||
"INCLUDE (duration_seconds, unavailable_since, via_explore, is_short, live_status)",
|
||||
"ix_videos_channel_pub_desc",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.get_context().autocommit_block():
|
||||
_swap(
|
||||
"ix_videos_channel_pub_desc",
|
||||
"ON videos (channel_id, published_at DESC NULLS LAST, id ASC)",
|
||||
"ix_videos_channel_pub_cover",
|
||||
)
|
||||
@@ -0,0 +1,86 @@
|
||||
"""videos: an index per feed sort, so a sorted page stops after its rows (E31 S4e)
|
||||
|
||||
The feed's views, duration and title sorts had no index. Every page (the first and each next cursor)
|
||||
read the whole videos heap, filtered it and kept a top-N: on prod ~350 ms a page for views/duration,
|
||||
550–780 ms for title (the ICU comparison on every row), 2.3 s for page 2 when the copies of one request
|
||||
queued. With an index in the exact ORDER BY (`feedquery.order_by`: the key, NULLS LAST, then id) the
|
||||
planner walks it and stops after the page. On a prod-copy localdev every plain sort fell from 180–310 ms
|
||||
to ~1 ms, deep pages (20) to 3–5 ms, and narrow filters (a tag, a channel, a year, nothing found) stayed
|
||||
at 0.3–25 ms.
|
||||
|
||||
One index per direction: a backward scan of `DESC NULLS LAST` gives `ASC NULLS FIRST`, not the
|
||||
`ASC NULLS LAST` the asc sorts ask for. The title pair is per reader language (`app.collation.ICU`):
|
||||
an index only serves an ORDER BY in its own collation.
|
||||
|
||||
Cost: ~130 MB (localdev: 10 + 10 + 8 + 8 + 4 × 23). Writes stay cheap: view_count and duration are
|
||||
written by the enrich, which rewrites indexed columns already (the revalidation job only touches
|
||||
last_checked_at), so no HOT update is lost. A search combined with one of these sorts can now walk the
|
||||
index too: a term matching ~2 % of the videos took 130–200 ms instead of 7 (still under the plain sort it
|
||||
replaces); rarer and broader terms stay fast. Fencing the index off under a search made broad terms
|
||||
(`m`, `the`) take 2 s, so it is left to the planner.
|
||||
|
||||
The title indexes are the schema's first ones on an ICU collation: an ICU upgrade in the Postgres image
|
||||
reorders them, and `app.collation.heal_drift` rebuilds them at startup when it sees one.
|
||||
tests/test_feed_sort_indexes.py holds each sort and its index together.
|
||||
|
||||
CONCURRENTLY (autocommit_block) builds them without blocking a live prod; IF (NOT) EXISTS keeps the drift
|
||||
test's downgrade → re-upgrade round trip idempotent.
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
revision: str = "0096_videos_sort_indexes"
|
||||
down_revision: str | None = "0095_videos_channel_cover_index"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
# name → the indexed ORDER BY (feedquery.sort_keys + order_by: key NULLS LAST, then id).
|
||||
INDEXES = {
|
||||
"ix_videos_views_desc": "view_count DESC NULLS LAST, id",
|
||||
"ix_videos_views_asc": "view_count ASC NULLS LAST, id",
|
||||
"ix_videos_duration_desc": "duration_seconds DESC NULLS LAST, id",
|
||||
"ix_videos_duration_asc": "duration_seconds ASC NULLS LAST, id",
|
||||
"ix_videos_title_en": '(lower(title) COLLATE "en-x-icu") ASC NULLS LAST, id',
|
||||
"ix_videos_title_en_desc": '(lower(title) COLLATE "en-x-icu") DESC NULLS LAST, id',
|
||||
"ix_videos_title_hu": '(lower(title) COLLATE "hu-x-icu") ASC NULLS LAST, id',
|
||||
"ix_videos_title_hu_desc": '(lower(title) COLLATE "hu-x-icu") DESC NULLS LAST, id',
|
||||
}
|
||||
|
||||
|
||||
def _valid(name: str) -> bool | None:
|
||||
"""The index's `indisvalid`, or None when there is no index of that name."""
|
||||
row = (
|
||||
op.get_bind()
|
||||
.execute(
|
||||
text(
|
||||
"SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid "
|
||||
"WHERE c.relname = :name"
|
||||
),
|
||||
{"name": name},
|
||||
)
|
||||
.first()
|
||||
)
|
||||
return None if row is None else bool(row[0])
|
||||
|
||||
|
||||
def _build(name: str, ddl: str) -> None:
|
||||
"""CREATE INDEX CONCURRENTLY `name` (`ddl` is what follows the name), re-runnable. A build that
|
||||
failed part-way leaves an INVALID index under the name, and `IF NOT EXISTS` would keep it: the
|
||||
planner never uses it, and nothing says so (E31 S5 review). Such a leftover is dropped and built
|
||||
again."""
|
||||
if _valid(name) is False:
|
||||
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||
op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} {ddl}")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.get_context().autocommit_block():
|
||||
for name, columns in INDEXES.items():
|
||||
_build(name, f"ON videos ({columns})")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.get_context().autocommit_block():
|
||||
for name in INDEXES:
|
||||
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||
@@ -0,0 +1,69 @@
|
||||
"""messages: drop direct messaging (the owner's decision, 2026-10-04)
|
||||
|
||||
Chat — end-to-end encrypted direct messages plus the server's welcome note — was never used: prod held
|
||||
two welcome notes, no message between people, and one key. Its tables go with its code. The rows are
|
||||
not migrated anywhere; the nightly backup holds them.
|
||||
|
||||
The downgrade restores the schema as it stood (0026 + 0027), empty: a message can't be brought back
|
||||
from a migration, and the E2EE key blobs were only ever readable by their owner's browser anyway.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0097_drop_messages"
|
||||
down_revision: str | None = "0096_videos_sort_indexes"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_table("message_keys")
|
||||
op.drop_index("ix_messages_created_at", table_name="messages")
|
||||
op.drop_index("ix_messages_recipient_id", table_name="messages")
|
||||
op.drop_index("ix_messages_sender_id", table_name="messages")
|
||||
op.drop_table("messages")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.create_table(
|
||||
"messages",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"sender_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=True
|
||||
),
|
||||
sa.Column(
|
||||
"recipient_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("body", sa.Text(), nullable=True),
|
||||
sa.Column("read_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.Column("kind", sa.String(length=16), nullable=False, server_default="user"),
|
||||
sa.Column("ciphertext", sa.Text(), nullable=True),
|
||||
sa.Column("iv", sa.String(length=32), nullable=True),
|
||||
)
|
||||
op.create_index("ix_messages_sender_id", "messages", ["sender_id"])
|
||||
op.create_index("ix_messages_recipient_id", "messages", ["recipient_id"])
|
||||
op.create_index("ix_messages_created_at", "messages", ["created_at"])
|
||||
op.create_table(
|
||||
"message_keys",
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
),
|
||||
sa.Column("public_key", sa.Text(), nullable=False),
|
||||
sa.Column("wrapped_private_key", sa.Text(), nullable=False),
|
||||
sa.Column("salt", sa.String(length=64), nullable=False),
|
||||
sa.Column("wrap_iv", sa.String(length=32), nullable=False),
|
||||
sa.Column("key_check", sa.Text(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,55 @@
|
||||
"""demo: drop the shared demo account (the owner's decision, 2026-10-04)
|
||||
|
||||
The demo account — one shared user that whitelisted emails entered from the login page, reset on a
|
||||
schedule — goes with its code. Prod held the demo user and 10 whitelist rows.
|
||||
|
||||
The upgrade deletes the demo user first (its playlists, states and subscriptions cascade; audit and
|
||||
quota rows keep their history with the actor nulled, as for any deleted account), then the traces
|
||||
that only the demo code could render: its audit rows (whitelist edits, manual resets, and every row
|
||||
about the reset job — its runs and its interval changes), the reset job's interval override and its
|
||||
completion notices. Nothing is migrated anywhere; the nightly backup holds the rows.
|
||||
|
||||
The downgrade restores the schema as it stood (0014), empty: the demo user is created lazily on its
|
||||
first sign-in, so an empty `demo_whitelist` is the whole of its state.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0098_drop_demo_account"
|
||||
down_revision: str | None = "0097_drop_messages"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("DELETE FROM users WHERE is_demo")
|
||||
op.execute(
|
||||
"DELETE FROM audit_log WHERE action IN ('demo_add', 'demo_remove', 'demo_reset')"
|
||||
" OR (target_type = 'job' AND target_id = 'demo_reset')"
|
||||
)
|
||||
op.execute("DELETE FROM scheduler_settings WHERE job_id = 'demo_reset'")
|
||||
op.execute("DELETE FROM notifications WHERE data->>'job_id' = 'demo_reset'")
|
||||
op.drop_index("ix_demo_whitelist_email", table_name="demo_whitelist")
|
||||
op.drop_table("demo_whitelist")
|
||||
op.drop_index("ix_users_is_demo", table_name="users")
|
||||
op.drop_column("users", "is_demo")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column(
|
||||
"users",
|
||||
sa.Column("is_demo", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
)
|
||||
op.create_index("ix_users_is_demo", "users", ["is_demo"])
|
||||
op.create_table(
|
||||
"demo_whitelist",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("email", sa.String(length=320), nullable=False),
|
||||
sa.Column("note", sa.Text(), nullable=True),
|
||||
sa.Column("added_by", sa.String(length=320), nullable=True),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
)
|
||||
op.create_index("ix_demo_whitelist_email", "demo_whitelist", ["email"], unique=True)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""library: a resume point per user and library item
|
||||
|
||||
The library player opened every item at 0:00. `library_states` keeps where a user stopped, per job
|
||||
(their own download, or one shared with them) — the same shape as `video_states`' position, without a
|
||||
status: the library has no "watched". It starts empty; the downgrade drops it.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0099_library_states"
|
||||
down_revision: str | None = "0098_drop_demo_account"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"library_states",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"job_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("download_jobs.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("position_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("progress_updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),
|
||||
)
|
||||
op.create_index("ix_library_states_job_id", "library_states", ["job_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_library_states_job_id", table_name="library_states")
|
||||
op.drop_table("library_states")
|
||||
@@ -0,0 +1,22 @@
|
||||
"""plex_items: drop `markers`, a column nothing writes or reads
|
||||
|
||||
Intro/credits markers are read live from Plex for the item the player opens (with Plex's `final` flag
|
||||
on the block that ends it); this cached copy was never filled — every row is NULL. The downgrade
|
||||
restores the empty column.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0100_drop_plex_item_markers"
|
||||
down_revision: str | None = "0099_library_states"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_column("plex_items", "markers")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column("plex_items", sa.Column("markers", sa.JSON(), nullable=True))
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Plex summaries and taglines as plain text
|
||||
|
||||
Revision ID: 0101_plex_text_plain
|
||||
Revises: 0100_drop_plex_item_markers
|
||||
Create Date: 2026-10-08
|
||||
|
||||
Some Plex agents write a summary as HTML (`<p>…</p>`, `á`), and the app showed the tags as
|
||||
text. The sync now stores every summary and tagline through `plain_text` (app/plex/plaintext.py);
|
||||
this cleans the rows it wrote before, through the same function, so nothing waits for a full resync.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from app.plex.plaintext import plain_text
|
||||
|
||||
revision: str = "0101_plex_text_plain"
|
||||
down_revision: str | None = "0100_drop_plex_item_markers"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
_COLUMNS = {
|
||||
"plex_items": ("summary", "summary_hu", "tagline", "tagline_hu"),
|
||||
"plex_shows": ("summary", "summary_hu", "tagline", "tagline_hu"),
|
||||
"plex_collections": ("summary",),
|
||||
}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
for table, columns in _COLUMNS.items():
|
||||
for col in columns:
|
||||
rows = conn.execute(
|
||||
sa.text(f"SELECT id, {col} FROM {table} WHERE {col} ~ '[<&]'")
|
||||
).fetchall()
|
||||
updates = [
|
||||
{"id": rid, "v": clean}
|
||||
for rid, value in rows
|
||||
if (clean := plain_text(value)) != value
|
||||
]
|
||||
if updates:
|
||||
conn.execute(sa.text(f"UPDATE {table} SET {col} = :v WHERE id = :id"), updates)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass # the markup is not wanted back; the next sync would clean it again
|
||||
@@ -0,0 +1,193 @@
|
||||
"""The Files library: folders, and a file's retention on the user's job
|
||||
|
||||
Revision ID: 0102_file_folders
|
||||
Revises: 0101_plex_text_plain
|
||||
Create Date: 2026-10-11
|
||||
|
||||
Every download, clip and upload of a user now sits in a folder of their Files library
|
||||
(`app.downloads.folders`): a source folder made per origin (YouTube, Plex, uploads, one per other
|
||||
site) or one of their own. How long a file stays moves from the shared asset to each user's job
|
||||
(`download_jobs.expires_at`, stamped from the folder's `retention_days`), so one user keeping a file
|
||||
no longer depends on another's copy; the asset's own `expires_at` / `gc_notified` go.
|
||||
|
||||
This places every existing job in its source folder and carries each done job's expiry over from
|
||||
the asset it holds (an upload keeps none). The source-key rule is copied here, frozen, from
|
||||
`folders.source_key_for` as it is today — a migration must not follow the code's later changes.
|
||||
"""
|
||||
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0102_file_folders"
|
||||
down_revision: str | None = "0101_plex_text_plain"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
|
||||
_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
|
||||
_DEFAULT_RETENTION_DAYS = 30 # config.download_retention_days on the day this was written
|
||||
|
||||
|
||||
def _site_key(url: str) -> str:
|
||||
host = (urlsplit(url).hostname or "").lower().rstrip(".")
|
||||
for door in _FRONT_DOORS:
|
||||
if host.startswith(door) and host.count(".") > 1:
|
||||
host = host[len(door) :]
|
||||
break
|
||||
if host in _YOUTUBE_HOSTS:
|
||||
return "youtube"
|
||||
return f"site:{host or 'unknown'}"
|
||||
|
||||
|
||||
def _asset_key(kind: str | None, ref: str | None) -> str:
|
||||
if kind == "youtube":
|
||||
return "youtube"
|
||||
if kind == "url" and ref:
|
||||
return _site_key(ref)
|
||||
return "upload"
|
||||
|
||||
|
||||
def _key(job, jobs_by_id, assets) -> str:
|
||||
kind = job.source_kind
|
||||
if kind == "youtube":
|
||||
return "youtube"
|
||||
if kind == "upload":
|
||||
return "upload"
|
||||
if kind == "clip":
|
||||
spec = job.edit_spec or {}
|
||||
return "plex" if spec.get("source") == "plex" else "youtube"
|
||||
if kind == "url":
|
||||
return _site_key(job.source_ref)
|
||||
if kind == "edit":
|
||||
parent = jobs_by_id.get(job.source_job_id)
|
||||
if parent is not None and parent.id != job.id:
|
||||
return _key(parent, jobs_by_id, assets)
|
||||
asset = assets.get(job.source_asset_id)
|
||||
if asset is not None:
|
||||
return _asset_key(asset.source_kind, asset.source_ref)
|
||||
return "upload"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"file_folders",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("kind", sa.String(16), nullable=False),
|
||||
sa.Column("source_key", sa.String(260)), # "site:" + a host (<= 253)
|
||||
sa.Column("name", sa.String(255)),
|
||||
sa.Column("retention_days", sa.Integer()),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||
sa.CheckConstraint(
|
||||
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)",
|
||||
name="ck_file_folder_kind",
|
||||
),
|
||||
)
|
||||
op.create_index("ix_file_folders_user_id", "file_folders", ["user_id"])
|
||||
op.create_index(
|
||||
"uq_file_folder_own_name",
|
||||
"file_folders",
|
||||
["user_id", sa.text("lower(name)")],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("name IS NOT NULL"),
|
||||
)
|
||||
|
||||
op.add_column(
|
||||
"download_jobs",
|
||||
sa.Column(
|
||||
"folder_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("file_folders.id", ondelete="RESTRICT"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"download_jobs",
|
||||
sa.Column("kept", sa.Boolean(), server_default="false", nullable=False),
|
||||
)
|
||||
op.add_column("download_jobs", sa.Column("expires_at", sa.DateTime(timezone=True)))
|
||||
op.add_column(
|
||||
"download_jobs",
|
||||
sa.Column("expiry_warned", sa.Boolean(), server_default="false", nullable=False),
|
||||
)
|
||||
op.create_index("ix_download_jobs_folder_id", "download_jobs", ["folder_id"])
|
||||
op.create_index("ix_download_jobs_expires_at", "download_jobs", ["expires_at"])
|
||||
|
||||
conn = op.get_bind()
|
||||
raw = conn.execute(
|
||||
sa.text("SELECT value FROM system_config WHERE key = 'download_retention_days'")
|
||||
).scalar()
|
||||
retention = int(raw) if raw not in (None, "") else _DEFAULT_RETENTION_DAYS
|
||||
|
||||
jobs = conn.execute(
|
||||
sa.text(
|
||||
"SELECT j.id, j.user_id, j.source_kind, j.source_ref, j.edit_spec, j.source_job_id,"
|
||||
" j.source_asset_id, j.status, j.asset_id, a.expires_at AS asset_expires_at"
|
||||
" FROM download_jobs j LEFT JOIN media_assets a ON a.id = j.asset_id"
|
||||
)
|
||||
).fetchall()
|
||||
jobs_by_id = {j.id: j for j in jobs}
|
||||
assets = {
|
||||
a.id: a
|
||||
for a in conn.execute(sa.text("SELECT id, source_kind, source_ref FROM media_assets"))
|
||||
}
|
||||
|
||||
folder_ids: dict[tuple[int, str], int] = {}
|
||||
for job in jobs:
|
||||
key = _key(job, jobs_by_id, assets)
|
||||
slot = (job.user_id, key)
|
||||
if slot not in folder_ids:
|
||||
folder_ids[slot] = conn.execute(
|
||||
sa.text(
|
||||
"INSERT INTO file_folders (user_id, kind, source_key, retention_days)"
|
||||
" VALUES (:u, 'source', :k, :r) RETURNING id"
|
||||
),
|
||||
{"u": job.user_id, "k": key, "r": None if key == "upload" else retention},
|
||||
).scalar_one()
|
||||
# A done file keeps the expiry it had (its asset's); an upload had none and keeps none.
|
||||
expires = job.asset_expires_at if job.status == "done" and key != "upload" else None
|
||||
conn.execute(
|
||||
sa.text("UPDATE download_jobs SET folder_id = :f, expires_at = :e WHERE id = :id"),
|
||||
{"f": folder_ids[slot], "e": expires, "id": job.id},
|
||||
)
|
||||
|
||||
op.drop_column("media_assets", "expires_at")
|
||||
op.drop_column("media_assets", "gc_notified")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column("media_assets", sa.Column("expires_at", sa.DateTime(timezone=True)))
|
||||
op.add_column(
|
||||
"media_assets",
|
||||
sa.Column("gc_notified", sa.Boolean(), server_default="false", nullable=False),
|
||||
)
|
||||
# The asset goes when its last holder would: the latest of its holders' expiries (a holder that
|
||||
# never expires keeps the asset unexpired).
|
||||
op.execute(
|
||||
"UPDATE media_assets a SET expires_at = sub.e FROM ("
|
||||
" SELECT asset_id, CASE WHEN bool_or(expires_at IS NULL) THEN NULL ELSE max(expires_at) END"
|
||||
" AS e FROM download_jobs WHERE asset_id IS NOT NULL AND status = 'done' GROUP BY asset_id"
|
||||
") sub WHERE a.id = sub.asset_id AND a.source_kind <> 'upload'"
|
||||
)
|
||||
op.drop_index("ix_download_jobs_expires_at", table_name="download_jobs")
|
||||
op.drop_index("ix_download_jobs_folder_id", table_name="download_jobs")
|
||||
op.drop_column("download_jobs", "expiry_warned")
|
||||
op.drop_column("download_jobs", "expires_at")
|
||||
op.drop_column("download_jobs", "kept")
|
||||
op.drop_column("download_jobs", "folder_id")
|
||||
op.drop_index("uq_file_folder_own_name", table_name="file_folders")
|
||||
op.drop_index("ix_file_folders_user_id", table_name="file_folders")
|
||||
op.drop_table("file_folders")
|
||||
@@ -0,0 +1,32 @@
|
||||
"""The Files library: a file's SHA-256, and the expired rows retired
|
||||
|
||||
Revision ID: 0103_files_sha256
|
||||
Revises: 0102_file_folders
|
||||
Create Date: 2026-10-11
|
||||
|
||||
`media_assets.sha256` holds the file's hash once it is known (an upload is hashed while it streams
|
||||
in, any other file on the first ask), so a machine fetching a file can check what it got.
|
||||
|
||||
An expired file now leaves its user's library (`downloads.gc`). Before, an expired download stayed
|
||||
as a fileless "expired · download again" row (a done job with no asset). Those rows hold nothing on
|
||||
disk and the library no longer shows them, so they are deleted here rather than left behind
|
||||
unseen; their shares and public links go with them (ON DELETE CASCADE), and those already served
|
||||
nothing. The downgrade cannot bring them back.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0103_files_sha256"
|
||||
down_revision: str | None = "0102_file_folders"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("media_assets", sa.Column("sha256", sa.String(64)))
|
||||
op.execute("DELETE FROM download_jobs WHERE status = 'done' AND asset_id IS NULL")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("media_assets", "sha256")
|
||||
@@ -0,0 +1,38 @@
|
||||
"""A file's "Added" is when it became done
|
||||
|
||||
Revision ID: 0104_job_ready_at
|
||||
Revises: 0103_files_sha256
|
||||
Create Date: 2026-10-12
|
||||
|
||||
`download_jobs.ready_at` is when a job became done, and the Files library sorts and dates by it: a
|
||||
download that failed for days and then went through used to sit days down the list, under its
|
||||
request date. Every path to done stamps it, held by `ck_download_jobs_ready_at`.
|
||||
|
||||
A done job from before has no record of that moment (its "ready to watch" notice never named the
|
||||
job, so it cannot be matched back), so it takes its request date, which is where it already sat.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0104_job_ready_at"
|
||||
down_revision: str | None = "0103_files_sha256"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("download_jobs", sa.Column("ready_at", sa.DateTime(timezone=True)))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE download_jobs SET ready_at = created_at WHERE status = 'done'
|
||||
"""
|
||||
)
|
||||
op.create_check_constraint(
|
||||
"ck_download_jobs_ready_at", "download_jobs", "status <> 'done' OR ready_at IS NOT NULL"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint("ck_download_jobs_ready_at", "download_jobs", type_="check")
|
||||
op.drop_column("download_jobs", "ready_at")
|
||||
@@ -0,0 +1,37 @@
|
||||
"""A file remembers where it came from
|
||||
|
||||
Revision ID: 0105_job_source_key
|
||||
Revises: 0104_job_ready_at
|
||||
Create Date: 2026-10-12
|
||||
|
||||
`download_jobs.source_key` is the origin a file came from — the source folder it may go back to —
|
||||
recorded when it is placed. An edit used to work its origin out through its parent each time, and
|
||||
once the parent was deleted or expired it fell back to "upload": moved into an own folder, it could
|
||||
no longer go back to its folder, and deleting that folder sent it to Uploads to stay forever.
|
||||
|
||||
A file in a source folder takes that folder's key. One already in an own folder is left NULL and
|
||||
keeps being worked out (`folders.source_key_for`), as before.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0105_job_source_key"
|
||||
down_revision: str | None = "0104_job_ready_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("download_jobs", sa.Column("source_key", sa.String(260)))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE download_jobs j SET source_key = f.source_key
|
||||
FROM file_folders f
|
||||
WHERE f.id = j.folder_id AND f.kind = 'source'
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("download_jobs", "source_key")
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Machine access to the Files library: API tokens bound to folders, and the Hub folder
|
||||
|
||||
Revision ID: 0106_api_tokens
|
||||
Revises: 0105_job_source_key
|
||||
Create Date: 2026-10-14
|
||||
|
||||
`api_tokens` is a user's token for a machine (a script, the hub): only its SHA-256 is kept, plus the
|
||||
first characters to recognise it by, and whether its user was told it is about to run out. `api_token_grants` says what it may do in which folder — list,
|
||||
read, write, delete — so a token reaches nothing it was not given, folder by folder.
|
||||
|
||||
A third folder kind, `hub`: the one folder a user exchanges files with their hub machine through.
|
||||
Like a source folder its name is the client's to render; like an own folder a file may be moved in.
|
||||
One per user.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0106_api_tokens"
|
||||
down_revision: str | None = "0105_job_source_key"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
_KIND_OLD = (
|
||||
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
||||
)
|
||||
_KIND_NEW = _KIND_OLD + " OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_constraint("ck_file_folder_kind", "file_folders", type_="check")
|
||||
op.create_check_constraint("ck_file_folder_kind", "file_folders", _KIND_NEW)
|
||||
op.create_index(
|
||||
"uq_file_folder_hub",
|
||||
"file_folders",
|
||||
["user_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("kind = 'hub'"),
|
||||
)
|
||||
op.create_table(
|
||||
"api_tokens",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("name", sa.String(80), nullable=False),
|
||||
sa.Column("prefix", sa.String(16), nullable=False),
|
||||
sa.Column("token_hash", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("last_used_at", sa.DateTime(timezone=True)),
|
||||
sa.Column("expiry_warned", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"api_token_grants",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"token_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("api_tokens.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"folder_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("file_folders.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("can_list", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||
sa.Column("can_read", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||
sa.Column("can_write", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||
sa.Column("can_delete", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||
sa.UniqueConstraint("token_id", "folder_id", name="uq_api_token_grant"),
|
||||
sa.CheckConstraint(
|
||||
"can_list OR can_read OR can_write OR can_delete", name="ck_api_token_grant_some"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("api_token_grants")
|
||||
op.drop_table("api_tokens")
|
||||
op.drop_index("uq_file_folder_hub", table_name="file_folders")
|
||||
# A Hub folder may hold files (their foreign key RESTRICTs a delete): it becomes an own folder,
|
||||
# named "Hub <id>" — or "Hub <id> (2)", … when its user already has a folder of that name,
|
||||
# which `uq_file_folder_own_name` (per user, case-blind) would refuse.
|
||||
conn = op.get_bind()
|
||||
hubs = conn.execute(sa.text("SELECT id, user_id FROM file_folders WHERE kind = 'hub'")).all()
|
||||
for folder_id, user_id in hubs:
|
||||
name, n = f"Hub {folder_id}", 1
|
||||
while conn.execute(
|
||||
sa.text("SELECT 1 FROM file_folders WHERE user_id = :u AND lower(name) = lower(:n)"),
|
||||
{"u": user_id, "n": name},
|
||||
).first():
|
||||
n += 1
|
||||
name = f"Hub {folder_id} ({n})"
|
||||
conn.execute(
|
||||
sa.text("UPDATE file_folders SET kind = 'own', name = :n WHERE id = :i"),
|
||||
{"n": name, "i": folder_id},
|
||||
)
|
||||
op.drop_constraint("ck_file_folder_kind", "file_folders", type_="check")
|
||||
op.create_check_constraint("ck_file_folder_kind", "file_folders", _KIND_OLD)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Share-link tokens no longer stored in the clear
|
||||
|
||||
Revision ID: 0107_link_token_at_rest
|
||||
Revises: 0106_api_tokens
|
||||
Create Date: 2026-10-15
|
||||
|
||||
A `/watch` link's token is the credential itself, and it sat in `download_links.token` as written,
|
||||
so a copy of the database was every shared file. Now a link is found by the token's SHA-256
|
||||
(`token_hash`), and the token is kept only Fernet-encrypted (`token_enc`, TOKEN_ENCRYPTION_KEY) so its
|
||||
owner can copy the same link again. Without a key the copy is not kept: the link still opens, its
|
||||
address just cannot be shown again (`app/downloads/links.py`).
|
||||
|
||||
Every existing link is converted here, so the addresses already sent keep working. The downgrade
|
||||
writes the tokens back from their encrypted copies; a link with none left (no key, or a key since
|
||||
changed) gets a fresh random token, which is the same as revoking it.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import secrets
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from app import security
|
||||
|
||||
revision: str = "0107_link_token_at_rest"
|
||||
down_revision: str | None = "0106_api_tokens"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("download_links", sa.Column("token_hash", sa.String(64)))
|
||||
op.add_column("download_links", sa.Column("token_enc", sa.Text()))
|
||||
conn = op.get_bind()
|
||||
sealing = security.can_encrypt()
|
||||
for link_id, token in conn.execute(sa.text("SELECT id, token FROM download_links")).fetchall():
|
||||
conn.execute(
|
||||
sa.text("UPDATE download_links SET token_hash = :h, token_enc = :e WHERE id = :id"),
|
||||
{
|
||||
"id": link_id,
|
||||
"h": hashlib.sha256(token.encode()).hexdigest(),
|
||||
"e": security.encrypt(token) if sealing else None,
|
||||
},
|
||||
)
|
||||
op.alter_column("download_links", "token_hash", nullable=False)
|
||||
op.create_unique_constraint("uq_download_link_token_hash", "download_links", ["token_hash"])
|
||||
op.drop_constraint("uq_download_link_token", "download_links", type_="unique")
|
||||
op.drop_column("download_links", "token")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column("download_links", sa.Column("token", sa.String(64)))
|
||||
conn = op.get_bind()
|
||||
rows = conn.execute(sa.text("SELECT id, token_enc FROM download_links")).fetchall()
|
||||
for link_id, enc in rows:
|
||||
token = security.decrypt_optional(enc) or secrets.token_urlsafe(24)
|
||||
conn.execute(
|
||||
sa.text("UPDATE download_links SET token = :t WHERE id = :id"),
|
||||
{"id": link_id, "t": token},
|
||||
)
|
||||
op.alter_column("download_links", "token", nullable=False)
|
||||
op.create_unique_constraint("uq_download_link_token", "download_links", ["token"])
|
||||
op.drop_constraint("uq_download_link_token_hash", "download_links", type_="unique")
|
||||
op.drop_column("download_links", "token_enc")
|
||||
op.drop_column("download_links", "token_hash")
|
||||
@@ -0,0 +1,40 @@
|
||||
"""A share link may open a folder
|
||||
|
||||
Revision ID: 0108_folder_links
|
||||
Revises: 0107_link_token_at_rest
|
||||
Create Date: 2026-10-15
|
||||
|
||||
A `/watch` link opened one file. Now it may open one of its owner's own folders instead: the link's
|
||||
page lists what is in the folder when it is opened. `folder_id` sits beside `job_id`, exactly one of
|
||||
the two set; a folder's links go with the folder.
|
||||
|
||||
The downgrade has nowhere to keep a folder link, so it deletes them: they stop opening.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0108_folder_links"
|
||||
down_revision: str | None = "0107_link_token_at_rest"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"download_links",
|
||||
sa.Column("folder_id", sa.Integer(), sa.ForeignKey("file_folders.id", ondelete="CASCADE")),
|
||||
)
|
||||
op.create_index("ix_download_links_folder_id", "download_links", ["folder_id"])
|
||||
op.alter_column("download_links", "job_id", nullable=True)
|
||||
op.create_check_constraint(
|
||||
"ck_download_link_target", "download_links", "(job_id IS NULL) <> (folder_id IS NULL)"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DELETE FROM download_links WHERE folder_id IS NOT NULL")
|
||||
op.drop_constraint("ck_download_link_target", "download_links", type_="check")
|
||||
op.alter_column("download_links", "job_id", nullable=False)
|
||||
op.drop_index("ix_download_links_folder_id", table_name="download_links")
|
||||
op.drop_column("download_links", "folder_id")
|
||||
+12
-8
@@ -1,7 +1,7 @@
|
||||
# Backend conventions
|
||||
|
||||
The rules below are already followed across `backend/app` — they lived only in a few module
|
||||
docstrings (`demo.py`, `userscope.py`, `routes/_common.py`) until this file collected them (E3 S3,
|
||||
docstrings (`userscope.py`, `routes/_common.py`) until this file collected them (E3 S3,
|
||||
B-core-20). When you add code, match these so the next reader doesn't have to guess "which pattern do
|
||||
I copy".
|
||||
|
||||
@@ -20,10 +20,12 @@ Three layers, by *who* runs the code — not by feature:
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline;
|
||||
`issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker.
|
||||
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth` (with `apitokens`, a machine's
|
||||
folder-scoped way into the Files library), `sysconfig`, `quota`, `audit`,
|
||||
`notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||
level) and `debugsession` (a recording that tags its user's lines, E16), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
||||
reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
module must never import a `routes/…` module).
|
||||
|
||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||
@@ -71,7 +73,9 @@ mutation it records, so an audit entry can never exist without its change (or vi
|
||||
## The gate
|
||||
|
||||
Everything below must be green before a change is done (run via `siftlode check`):
|
||||
`ruff` (backend) · `tsc` ×3 + `eslint` + `prettier` + `knip` (frontend) · `vitest` · `pytest`. The
|
||||
pytest lane runs in `backend/docker-compose.test.yml` — an ephemeral Postgres + the backend tree
|
||||
bind-mounted — so the DB-backed integration tests (the `db` fixture) actually run; `SIFTLODE_TEST_
|
||||
REQUIRE_DB=1` there turns a would-be silent skip into a hard failure. See `docs/development.md`.
|
||||
`ruff` + `basedpyright` (backend) · the lock check + `tsc` ×3 + `eslint` + `prettier` + `knip` +
|
||||
`vite build` (frontend) · `vitest` · `pytest`. The pytest lane runs in `backend/docker-compose.test.yml`
|
||||
— an ephemeral Postgres + the backend tree bind-mounted — so the DB-backed integration tests (the `db`
|
||||
fixture) actually run; a missing database is a hard failure, not a silent skip, unless a hand run asks
|
||||
for pure-logic only with `SIFTLODE_TEST_REQUIRE_DB=0`. See `docs/development.md`, and `docs/gates.md`
|
||||
for what each gate deliberately leaves out.
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Machine access to the Files library: API tokens, and what each one may do where.
|
||||
|
||||
A user makes a token for a machine of theirs — their hub, a script — and it reaches their library as
|
||||
`Authorization: Bearer sfl_…`. Only on `/api/files`: the routes there take `files_caller` instead of
|
||||
`current_user`, and nothing else does (a test holds the route table to that), so a token can never
|
||||
manage the account, the tokens, or anything outside the library. And only in the folders its grants
|
||||
name, with the rights they give: list, read, write, delete. Every route asks the one question,
|
||||
`FilesCaller.may(folder_id, right)`; a signed-in user may everything in their own folders.
|
||||
|
||||
The token is `sfl_` + 32 random bytes in base62 + a CRC32 of them: a mistyped or cut-off token is
|
||||
refused without a database read, and a secret scanner can tell one by its shape. Only its SHA-256 is
|
||||
kept (`security.hash_token`) — the user sees it once, when it is made — with its first characters to
|
||||
tell tokens apart by. It is looked up by that hash on every request: no signed claims, so a deleted
|
||||
token stops working at once. A token about to run out warns its user once, a week before
|
||||
(`warn_expiring`), so a machine does not lose its key unannounced.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
import zlib
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import StrEnum
|
||||
|
||||
from fastapi import Depends, Request
|
||||
from sqlalchemy import delete, or_, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, debugsession, logs
|
||||
from app.audit import AuditAction
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import ApiToken, ApiTokenGrant, User
|
||||
from app.notifications import create_notification
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.refusal import refused
|
||||
from app.security import hash_token
|
||||
from app.utils import iso
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
PREFIX = "sfl_"
|
||||
_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
_BODY_BYTES = 32
|
||||
_BODY_LEN = 43 # 62**43 > 2**256: every 32-byte value fits
|
||||
_CHECK_LEN = 6 # 62**6 > 2**32: every CRC32 fits
|
||||
TOKEN_LEN = len(PREFIX) + _BODY_LEN + _CHECK_LEN
|
||||
# What a token is shown by once it is made: enough to tell a user's tokens apart, far too little to
|
||||
# narrow a guess (8 of its 43 random characters).
|
||||
SHOWN_LEN = len(PREFIX) + 8
|
||||
|
||||
# A machine that runs away (a loop in a script) is held back per token; a caller trying tokens is
|
||||
# held back per client, counting only the refusals.
|
||||
_calls = RateLimiter(600, 60)
|
||||
_bad = RateLimiter(20, 300)
|
||||
# How stale `last_used_at` may get before a request writes it again: a busy script costs one write a
|
||||
# minute, not one per request.
|
||||
_TOUCH_EVERY = timedelta(minutes=1)
|
||||
# How long before a token runs out its user is told (`warn_expiring`); the token list marks it
|
||||
# "expires soon" for the same span.
|
||||
EXPIRY_WARNING = timedelta(days=7)
|
||||
|
||||
|
||||
class Right(StrEnum):
|
||||
LIST = "list"
|
||||
READ = "read"
|
||||
WRITE = "write"
|
||||
DELETE = "delete"
|
||||
|
||||
|
||||
# Each right's column on a grant. A dict over the enum, so a new right without a column fails here.
|
||||
GRANT_COLUMNS = {
|
||||
Right.LIST: "can_list",
|
||||
Right.READ: "can_read",
|
||||
Right.WRITE: "can_write",
|
||||
Right.DELETE: "can_delete",
|
||||
}
|
||||
|
||||
|
||||
def _b62(value: int, width: int) -> str:
|
||||
out = []
|
||||
for _ in range(width):
|
||||
value, digit = divmod(value, 62)
|
||||
out.append(_ALPHABET[digit])
|
||||
return "".join(reversed(out))
|
||||
|
||||
|
||||
def _check(body: str) -> str:
|
||||
return _b62(zlib.crc32(body.encode()), _CHECK_LEN)
|
||||
|
||||
|
||||
def mint() -> str:
|
||||
"""A new token, in full. The caller stores its hash and shows it to the user, once."""
|
||||
body = _b62(int.from_bytes(secrets.token_bytes(_BODY_BYTES)), _BODY_LEN)
|
||||
return PREFIX + body + _check(body)
|
||||
|
||||
|
||||
def well_formed(raw: str) -> bool:
|
||||
"""Whether `raw` has a token's shape and its checksum adds up."""
|
||||
if len(raw) != TOKEN_LEN or not raw.startswith(PREFIX):
|
||||
return False
|
||||
body, check = raw[len(PREFIX) : -_CHECK_LEN], raw[-_CHECK_LEN:]
|
||||
# The check too: compare_digest refuses (TypeError, a 500) a str with anything non-ASCII in it.
|
||||
if any(c not in _ALPHABET for c in body + check):
|
||||
return False
|
||||
return secrets.compare_digest(_check(body), check)
|
||||
|
||||
|
||||
def grant_rights(grant: ApiTokenGrant) -> frozenset[Right]:
|
||||
return frozenset(r for r, col in GRANT_COLUMNS.items() if getattr(grant, col))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FilesCaller:
|
||||
"""Who is asking the Files library, and where they may do what. `grants` is None for a signed-in
|
||||
user (everything in their own folders); for a token, folder id → its rights there."""
|
||||
|
||||
user: User
|
||||
token_id: int | None = None
|
||||
token_prefix: str | None = None
|
||||
grants: Mapping[int, frozenset[Right]] | None = None
|
||||
|
||||
def may(self, folder_id: int | None, right: Right) -> bool:
|
||||
if self.grants is None:
|
||||
return True
|
||||
return folder_id is not None and right in self.grants.get(folder_id, frozenset())
|
||||
|
||||
def folders_with(self, right: Right) -> set[int] | None:
|
||||
"""The folders `right` is granted in; None = any of the user's own."""
|
||||
if self.grants is None:
|
||||
return None
|
||||
return {f for f, rights in self.grants.items() if right in rights}
|
||||
|
||||
@property
|
||||
def via(self) -> str | None:
|
||||
"""How an audit row says a token did it, None when the user did."""
|
||||
return None if self.token_prefix is None else f"api token {self.token_prefix}…"
|
||||
|
||||
|
||||
def bearer_token(request: Request) -> str | None:
|
||||
"""The token of an `Authorization: Bearer` header; None when the request carries none (another
|
||||
scheme counts as none — it is not ours to judge)."""
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
if scheme.lower() != "bearer":
|
||||
return None
|
||||
return value.strip()
|
||||
|
||||
|
||||
def session_caller(request: Request, db: Session = Depends(get_db)) -> User | None:
|
||||
"""The signed-in user, or None when the request brings a token instead: a token wins over a
|
||||
cookie, so a machine's request is judged by its grants alone, whatever else it carries."""
|
||||
if bearer_token(request) is not None:
|
||||
return None
|
||||
return current_user(request, db)
|
||||
|
||||
|
||||
def files_caller(
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
user: User | None = Depends(session_caller),
|
||||
) -> FilesCaller:
|
||||
"""The `/api/files` caller: a signed-in user, or a valid token's user with its grants."""
|
||||
if user is not None:
|
||||
return FilesCaller(user=user)
|
||||
return _token_caller(request, db, bearer_token(request) or "")
|
||||
|
||||
|
||||
def _refuse_token(key: str, reason: str) -> Exception:
|
||||
_bad.allow(key)
|
||||
log.info(
|
||||
"a request brought an API token that does not hold",
|
||||
extra=logs.event("apitoken.refused", reason=reason),
|
||||
)
|
||||
return refused(401, "not_authenticated")
|
||||
|
||||
|
||||
def _token_caller(request: Request, db: Session, raw: str) -> FilesCaller:
|
||||
key = client_key(request)
|
||||
if _bad.exhausted(key):
|
||||
raise refused(429, "api_token_rate_limited")
|
||||
if not well_formed(raw):
|
||||
raise _refuse_token(key, "malformed")
|
||||
token = db.execute(
|
||||
select(ApiToken).where(ApiToken.token_hash == hash_token(raw))
|
||||
).scalar_one_or_none()
|
||||
if token is None:
|
||||
raise _refuse_token(key, "unknown")
|
||||
now = datetime.now(timezone.utc)
|
||||
if token.expires_at is not None and token.expires_at <= now:
|
||||
raise _refuse_token(key, "expired")
|
||||
user = db.get(User, token.user_id)
|
||||
if user is None or not user.is_active or user.is_suspended:
|
||||
raise _refuse_token(key, "account")
|
||||
if not _calls.allow(str(token.id)):
|
||||
raise refused(429, "api_token_rate_limited")
|
||||
grants = {
|
||||
g.folder_id: grant_rights(g)
|
||||
for g in db.execute(select(ApiTokenGrant).where(ApiTokenGrant.token_id == token.id))
|
||||
.scalars()
|
||||
.all()
|
||||
}
|
||||
caller = FilesCaller(user=user, token_id=token.id, token_prefix=token.prefix, grants=grants)
|
||||
db.execute(
|
||||
update(ApiToken)
|
||||
.where(
|
||||
ApiToken.id == token.id,
|
||||
or_(ApiToken.last_used_at.is_(None), ApiToken.last_used_at < now - _TOUCH_EVERY),
|
||||
)
|
||||
.values(last_used_at=now)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
# Committed here, ahead of the route: a read-only route never commits, and a write route must
|
||||
# not carry this along into a rollback of its own.
|
||||
db.commit()
|
||||
logs.set_user(user.id, *debugsession.level_for(user, None))
|
||||
request.state.user_id = user.id
|
||||
return caller
|
||||
|
||||
|
||||
def revoke_all(db: Session, user_id: int) -> int:
|
||||
"""Revoke every token of `user_id`, each with its audit row (NOT committed). A token is a
|
||||
credential like a session, so whatever ends every session of an account — a password reset or
|
||||
change, "log out everywhere", a revoked untrusted password — ends its tokens too: one minted
|
||||
from a stolen session must not outlive the owner's answer to that theft. Returns how many."""
|
||||
gone = (
|
||||
db.execute(delete(ApiToken).where(ApiToken.user_id == user_id).returning(ApiToken.id))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for token_id in gone:
|
||||
audit.record(
|
||||
db,
|
||||
user_id,
|
||||
AuditAction.API_TOKEN_REVOKE,
|
||||
target_type="api_token",
|
||||
target_id=token_id,
|
||||
summary="revoked with every session of the account",
|
||||
)
|
||||
return len(gone)
|
||||
|
||||
|
||||
def expiring(token: ApiToken, now: datetime) -> bool:
|
||||
"""Whether `token` runs out within the warning span (and has not yet)."""
|
||||
return token.expires_at is not None and now < token.expires_at <= now + EXPIRY_WARNING
|
||||
|
||||
|
||||
def warn_expiring(db: Session) -> int:
|
||||
"""Tell each user whose token runs out within `EXPIRY_WARNING` — once per token: the mark and the
|
||||
notice commit together, and a token made again (a new row) warns afresh. Returns how many."""
|
||||
now = datetime.now(timezone.utc)
|
||||
hits = db.execute(
|
||||
update(ApiToken)
|
||||
.where(
|
||||
ApiToken.expiry_warned.is_(False),
|
||||
ApiToken.expires_at.is_not(None),
|
||||
ApiToken.expires_at > now,
|
||||
ApiToken.expires_at <= now + EXPIRY_WARNING,
|
||||
)
|
||||
.values(expiry_warned=True)
|
||||
.returning(ApiToken.id, ApiToken.user_id, ApiToken.name, ApiToken.expires_at)
|
||||
.execution_options(synchronize_session=False)
|
||||
).all()
|
||||
for token_id, user_id, name, expires_at in hits:
|
||||
create_notification(
|
||||
db,
|
||||
user_id,
|
||||
"api_token_expiring",
|
||||
title=f"API token {name} expires soon", # English fallback; the client names it from data
|
||||
data={"token_id": token_id, "name": name, "expires_at": iso(expires_at)},
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
return len(hits)
|
||||
+24
-6
@@ -27,7 +27,7 @@ class AuditAction:
|
||||
USER_SUSPEND = "user_suspend"
|
||||
USER_UNSUSPEND = "user_unsuspend"
|
||||
USER_DELETE = "user_delete"
|
||||
# E9.5 S5: an admin let a stuck registrant in, or re-sent their confirmation link. Both are
|
||||
# An admin let a stuck registrant in, or re-sent their confirmation link. Both are
|
||||
# recorded because both are the admin acting ON someone's access, not merely reading a list.
|
||||
USER_ACTIVATE = "user_activate"
|
||||
USER_VERIFY_RESEND = "user_verify_resend"
|
||||
@@ -36,12 +36,9 @@ class AuditAction:
|
||||
INVITE_DENY = "invite_deny"
|
||||
INVITE_ADD = "invite_add"
|
||||
INVITE_DELETE = "invite_delete"
|
||||
# E9.5 S6: the admin chose to tell someone their address is whitelisted. Adding the address is
|
||||
# The admin chose to tell someone their address is whitelisted. Adding the address is
|
||||
# silent, so this is the act that actually reaches a person — and the one worth a record.
|
||||
INVITE_SEND = "invite_send"
|
||||
DEMO_ADD = "demo_add"
|
||||
DEMO_REMOVE = "demo_remove"
|
||||
DEMO_RESET = "demo_reset"
|
||||
DISCOVERY_PURGE = "discovery_purge"
|
||||
# Config
|
||||
CONFIG_SET = "config_set"
|
||||
@@ -58,9 +55,30 @@ class AuditAction:
|
||||
DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete"
|
||||
# The log itself was cleared (a single row survives the clear as tamper-evidence)
|
||||
AUDIT_CLEAR = "audit_clear"
|
||||
# E16: a user sent a debug session's issue to the tracker — the one act that takes their log
|
||||
# A user sent a debug session's issue to the tracker — the one act that takes their log
|
||||
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
|
||||
ISSUE_CREATE = "issue_create"
|
||||
# The Files library (`routes.files`): what a user put in, took out, kept or reorganised. Ids
|
||||
# only — a file's or folder's name is the user's own text, not the admin's to read here.
|
||||
FILE_UPLOAD = "file_upload"
|
||||
FILE_DELETE = "file_delete"
|
||||
FILE_KEEP = "file_keep"
|
||||
FILE_UNKEEP = "file_unkeep"
|
||||
FILE_MOVE = "file_move"
|
||||
FILE_FOLDER_CREATE = "file_folder_create"
|
||||
FILE_FOLDER_RENAME = "file_folder_rename"
|
||||
FILE_FOLDER_RETENTION = "file_folder_retention"
|
||||
FILE_FOLDER_DELETE = "file_folder_delete"
|
||||
# A machine's token to the library (`routes.tokens`): made (its folders and rights) or revoked.
|
||||
# What a token then did is the file rows above, their summary naming it.
|
||||
API_TOKEN_CREATE = "api_token_create"
|
||||
API_TOKEN_REVOKE = "api_token_revoke"
|
||||
# A public `/watch` link to a file or a folder (`routes.downloads`): made, its settings changed,
|
||||
# revoked.
|
||||
# Never its token or password — only whether it has one.
|
||||
DOWNLOAD_LINK_CREATE = "download_link_create"
|
||||
DOWNLOAD_LINK_UPDATE = "download_link_update"
|
||||
DOWNLOAD_LINK_REVOKE = "download_link_revoke"
|
||||
|
||||
|
||||
def record(
|
||||
|
||||
+10
-47
@@ -24,8 +24,6 @@ def operator_contact() -> str | None:
|
||||
return next(iter(sorted(settings.admin_email_set)), None)
|
||||
|
||||
|
||||
DEMO_GOOGLE_SUB = "__demo__"
|
||||
DEMO_EMAIL = "demo@siftlode.local"
|
||||
MAX_SESSION_ACCOUNTS = 6
|
||||
|
||||
# OAuth scope vocabulary — the atoms + the request bundles, kept together as core. has_*_scope check
|
||||
@@ -69,7 +67,7 @@ def has_write_scope(user: User) -> bool:
|
||||
def env_allowed_canonicals() -> set[str]:
|
||||
"""The env bootstrap allow-list, in CANONICAL form — the only shape it may be compared in.
|
||||
|
||||
S3 canonicalised the Invite branch of `is_allowed` and left this one matching the literal string,
|
||||
Canonicalising only the Invite branch of `is_allowed` left this one matching the literal string,
|
||||
which is the 2026-09-03 lockout with a different source of truth: an operator who writes
|
||||
`first.last@gmail.com` into ALLOWED_EMAILS is refused when Google answers with the undotted
|
||||
primary, and a `+tag`ged entry can never match at all, because a Google account cannot contain a
|
||||
@@ -129,7 +127,7 @@ def can_activate(db: Session, user: User) -> bool:
|
||||
Lives in core so the admin surface and the /api/me badge count answer from ONE definition; a
|
||||
button that appears and a number that counts must never disagree about who is stuck.
|
||||
"""
|
||||
if user.is_demo or user.is_suspended or user.is_active:
|
||||
if user.is_suspended or user.is_active:
|
||||
return False
|
||||
return is_allowed(db, user.email)
|
||||
|
||||
@@ -147,7 +145,6 @@ def count_pending_activations(db: Session) -> int:
|
||||
rows = (
|
||||
db.execute(
|
||||
select(User).where(
|
||||
User.is_demo.is_(False),
|
||||
User.is_suspended.is_(False),
|
||||
User.is_active.is_(False),
|
||||
)
|
||||
@@ -188,24 +185,6 @@ def find_user_by_email(db: Session, email: str) -> User | None:
|
||||
return candidates[0] if len(candidates) == 1 else None
|
||||
|
||||
|
||||
def get_or_create_demo_user(db: Session) -> User:
|
||||
"""The one shared demo user, created on first use. No OAuth token / YouTube scope, so
|
||||
has_read_scope/has_write_scope are False and every YouTube-touching path is closed to it."""
|
||||
user = db.query(User).filter(User.is_demo.is_(True)).one_or_none()
|
||||
if user is None:
|
||||
user = User(
|
||||
google_sub=DEMO_GOOGLE_SUB,
|
||||
email=DEMO_EMAIL,
|
||||
display_name="Demo",
|
||||
role="user",
|
||||
is_demo=True,
|
||||
preferences={"language": "en"},
|
||||
)
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return user
|
||||
|
||||
|
||||
def revoke_google_token(token: str | None) -> None:
|
||||
"""Best-effort revocation of a Google OAuth grant at Google's endpoint. Revoking the refresh
|
||||
token tears down the whole grant, so a later sign-in starts from a clean consent (no scopes
|
||||
@@ -278,8 +257,7 @@ ACTIVE_ACCOUNT_HEADER = "x-siftlode-account"
|
||||
|
||||
def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int | None, bool]:
|
||||
"""Which account this connection acts as, and whether that's the session's default account.
|
||||
Takes any HTTPConnection — an HTTP Request OR a WebSocket — so the WS push channel shares this
|
||||
exact per-tab resolution instead of re-implementing it.
|
||||
Takes an HTTPConnection rather than a Request: it reads only the session and the headers.
|
||||
|
||||
The signed cookie holds the *wallet* (`account_ids`, every account signed into this browser)
|
||||
plus a default `user_id`. A caller may name its account with the X-Siftlode-Account header (or
|
||||
@@ -297,8 +275,8 @@ def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int
|
||||
"""
|
||||
default_id = conn.session.get("user_id")
|
||||
wallet = conn.session.get("account_ids") or []
|
||||
# Header for normal XHR; ?account= for contexts that can't set headers (WebSocket, and a
|
||||
# plain <a> file download). Both are wallet-gated below.
|
||||
# Header for normal XHR; ?account= for a context that can't set headers (a plain <a> file
|
||||
# download). Both are wallet-gated below.
|
||||
hdr = conn.headers.get(ACTIVE_ACCOUNT_HEADER) or conn.query_params.get("account")
|
||||
if hdr:
|
||||
try:
|
||||
@@ -362,12 +340,12 @@ def _session_user(request: Request, db: Session, user_id: int | None, is_default
|
||||
if default_id and default_id not in accounts:
|
||||
accounts.append(default_id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
# Attribute this request's log lines to the account (E22). Here, not in the request middleware:
|
||||
# Attribute this request's log lines to the account. Here, not in the request middleware:
|
||||
# this is the one place a session becomes an account, and the middleware runs outside both the
|
||||
# session middleware and this task's context. `request.state` is shared across middleware layers
|
||||
# (it lives on the ASGI scope), so the access-log line can read it back. The user's consented
|
||||
# diagnostics level rides along: their requests are logged — and stored — in that detail (S5).
|
||||
# So does a debug session the request names (E16): only the caller's own open one counts, and a
|
||||
# diagnostics level rides along: their requests are logged — and stored — in that detail.
|
||||
# So does a debug session the request names: only the caller's own open one counts, and a
|
||||
# header naming anything else is ignored rather than refused — a stale tab must keep working.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, until = debugsession.level_for(user, session)
|
||||
@@ -403,25 +381,10 @@ def optional_current_user(request: Request, db: Session = Depends(get_db)) -> Us
|
||||
return None
|
||||
|
||||
|
||||
def require_human(user: User = Depends(current_user)) -> User:
|
||||
"""Reject the shared demo account from actions that need a real Google/YouTube identity
|
||||
or spend the shared quota. Most YouTube paths are already closed to it implicitly (it has
|
||||
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
|
||||
where there's no scope check to lean on."""
|
||||
if user.is_demo:
|
||||
# Structured, because `req` leaves 403 to the caller and `notifyActionError` only shows a
|
||||
# STRUCTURED detail — a plain string here reached the demo user as their caller's generic
|
||||
# "couldn't do that", with no hint that the demo account is the reason (round 6). Affordances
|
||||
# this gate sits behind are not all UI-gated on `is_demo` (the comfort shelf is not).
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
return user
|
||||
|
||||
|
||||
def require_write_scope(user: User = Depends(require_human)) -> User:
|
||||
def require_write_scope(user: User = Depends(current_user)) -> User:
|
||||
"""Dependency: require the YouTube write scope (subscribe/unsubscribe, playlist export/edit).
|
||||
Single source of truth for the write gate — replaces the inline `if not has_write_scope(user)`
|
||||
checks that had drifted into three different 403 messages across channels/playlists. Layers on
|
||||
`require_human` because writing to YouTube needs a real, non-demo identity."""
|
||||
checks that had drifted into three different 403 messages across channels/playlists."""
|
||||
if not has_write_scope(user):
|
||||
raise refused(403, "youtube_editing_off")
|
||||
return user
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""The global request-body cap (SEC-20).
|
||||
|
||||
**The hole.** FastAPI reads a declared body IN FULL before auth or any other dependency runs — the
|
||||
body parameter has to be parsed to be passed in. So `POST /auth/password-login` drains a 32 MiB
|
||||
upload from an anonymous caller and only then answers 401. 70 of the 72 body-declaring routes behave
|
||||
the same way: one unauthenticated client can spend the process's memory and time at will.
|
||||
|
||||
**Why a middleware and not `capped_body` on every route.** `routes/_common.py:capped_body` is the
|
||||
right tool where a route genuinely takes bulk bytes, and the two routes that use it use it
|
||||
correctly. But putting it on all 70 others would (a) fight the canonical "Request body → a Pydantic
|
||||
model" row, since capping means taking the body as raw bytes and parsing it by hand, and (b) leave
|
||||
the NEXT route added uncovered, because nothing would remind its author. A default that has to be
|
||||
opted into is not a default. This module inverts it: the cap applies to everything, and the few
|
||||
routes that need more opt OUT, in one list, next to the code that reads it.
|
||||
|
||||
**Why raw ASGI and not `@app.middleware("http")`.** Starlette's BaseHTTPMiddleware cannot wrap
|
||||
`receive`, so it can only read `Content-Length` — which a client may omit entirely (chunked) or
|
||||
simply understate. The bytes have to be counted as they arrive, which is the same rule `capped_body`
|
||||
already applies per-route, and counting needs the receive channel.
|
||||
|
||||
**Why the refusal is byte-identical to `capped_body`'s.** Two shapes for one fact is how the
|
||||
`cancelled`/`canceled` bug happened. The 413 body here is built through the same `JSONResponse`
|
||||
FastAPI renders `HTTPException(413, {"code": "too_large"})` with — not a hand-rolled `json.dumps`,
|
||||
whose default separators differ by two bytes — and `tests/test_body_limit.py` compares the RAW BYTES
|
||||
of the two against each other rather than against a literal, so they cannot drift apart. The same
|
||||
reasoning decides where the middleware is registered (`main.py`): just inside CORS, so this 413
|
||||
carries the same headers `capped_body`'s does.
|
||||
|
||||
**No admin knob, deliberately.** The ceiling is a constant, not a `sysconfig` value: an attacker-
|
||||
facing limit that an admin can raise from the UI is a limit that can be raised by whoever gets the
|
||||
admin session, and nobody has asked to tune it. If a self-hoster ever needs a different ceiling it
|
||||
belongs in `config.py` (env, set before the process starts), not in the database.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from app import debugsession
|
||||
from app.config import settings
|
||||
|
||||
# 1 MiB. The largest body any route on the default ceiling legitimately takes is 512 KiB (the client
|
||||
# log sink, `routes/diagnostics.py:BODY_MAX_BYTES`), so this leaves 2x headroom — a ceiling that
|
||||
# refuses a legitimate request would be a worse bug than the one this module fixes. That ratio is
|
||||
# the load-bearing fact, so `test_body_limit.py` holds it rather than the figure above.
|
||||
DEFAULT_MAX_BODY_BYTES = 1024 * 1024
|
||||
|
||||
# The routes that need MORE than the default, as (method, path TEMPLATE) → their own ceiling. The
|
||||
# template, not a concrete path, so the list reads like the route decorators it mirrors and
|
||||
# `test_body_limit.py` can assert that each one still resolves to a real route: a renamed route
|
||||
# fails the gate instead of silently losing its allowance in prod.
|
||||
#
|
||||
# These are RAISED CEILINGS, NOT EXEMPTIONS, and the difference is the whole point. An exemption
|
||||
# would mean "this route is counted by something else" — which is only true when the route's own
|
||||
# limit applies to the bytes as they ARRIVE. That, in turn, is only true of a route with NO DECLARED
|
||||
# BODY PARAMETER: FastAPI parses and spools a declared body BEFORE the first dependency runs, so a
|
||||
# route that declares one has already stored the bytes by the time its own cap — or its auth — gets
|
||||
# a say.
|
||||
#
|
||||
# Both listed routes now satisfy that. The screenshot route always did (`capped_body` streams it,
|
||||
# and the owner check resolves first). The upload route did NOT at first: it declared
|
||||
# `file: UploadFile = File(...)`, so an anonymous caller could make the process write the whole
|
||||
# per-file allowance to temp before the auth dependency refused them — this middleware bounded that
|
||||
# number but could not stop the writing. It now reads its own body from a dependency declared after
|
||||
# the auth one, so its in-handler cap measures bytes as they arrive, like everything else here.
|
||||
#
|
||||
# That rule is no longer only written here: `test_body_limit.py` parses the routes and fails the
|
||||
# gate if any route in this dict declares a body parameter. A comment could not have stopped the
|
||||
# defect it describes; the test can.
|
||||
#
|
||||
# A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why
|
||||
# `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the
|
||||
# default, so listing it could only widen what it accepts.
|
||||
ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
||||
# Ingesting a local file. The body IS the file now — no multipart envelope to allow for —
|
||||
# so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s
|
||||
# `download_upload_max_bytes` is bounded by this same constant (see sysconfig.SPECS), so an
|
||||
# admin can only ever LOWER the route's cap below the edge, never raise it past one.
|
||||
#
|
||||
# WHICH LAYER ANSWERS, measured rather than assumed:
|
||||
# * knob LOWERED (an override below the default) — the route's cap is the smaller number, so
|
||||
# `capped_stream_to` trips first and answers 422 `upload_too_large` carrying the admin's own
|
||||
# figure, after draining the rest of the body so the client can read it.
|
||||
# * knob at the DEFAULT — the two numbers are EQUAL, and the edge always wins: a declared
|
||||
# Content-Length over the cap is refused here before a byte is read, and an undeclared body
|
||||
# trips this counter on the same chunk the route would have tripped on. The route's 422 is
|
||||
# unreachable in that configuration, and no amount of headroom would change it — the
|
||||
# declared-length check fires first for any body big enough to matter, and the route's drain
|
||||
# pushes the count past the edge for the rest.
|
||||
#
|
||||
# So this 413 is a refusal a real user meets on a default install, not a theoretical edge, and
|
||||
# it has to be as readable as the route's own: the client localizes `too_large` (`errors.tooLarge`)
|
||||
# exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before
|
||||
# sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports.
|
||||
("POST", "/api/files/upload"): settings.download_upload_max_bytes,
|
||||
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
|
||||
}
|
||||
|
||||
# The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders
|
||||
# as — built through the same JSONResponse so the separators match too, and asserted against a real
|
||||
# route's raw bytes in `test_body_limit.py` rather than against a literal here.
|
||||
TOO_LARGE_BODY = JSONResponse({"detail": {"code": "too_large"}}, status_code=413).body
|
||||
|
||||
_PATH_PARAM = re.compile(r"\{[^/{}]+\}")
|
||||
|
||||
|
||||
def template_to_regex(template: str) -> re.Pattern[str]:
|
||||
"""`/api/debug-sessions/{sid}/screenshots` → a regex matching one concrete path.
|
||||
|
||||
A path parameter matches `[^/]+` — one segment, never across a `/`, so `{sid}` cannot swallow a
|
||||
deeper path and hand an exemption to a route that was never granted one.
|
||||
"""
|
||||
literals = [re.escape(part) for part in _PATH_PARAM.split(template)]
|
||||
return re.compile("^" + "[^/]+".join(literals) + "$")
|
||||
|
||||
|
||||
def _declared_length(scope: Scope) -> int | None:
|
||||
"""The `Content-Length` header as an int, or None when absent or unparseable (a chunked body has
|
||||
none, and a malformed one is not a number we may trust — both fall through to counting)."""
|
||||
for name, value in scope.get("headers", ()):
|
||||
if name == b"content-length":
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
# silent-ok: a malformed Content-Length is the caller's doing, not a fault of ours,
|
||||
# and it is about to be answered anyway — returning None drops the request into the
|
||||
# counting path, which refuses on the real bytes. Logging it would hand an anonymous
|
||||
# client a free line in our log for every request it cares to malform.
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
async def _send_too_large(send: Send) -> None:
|
||||
await send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": 413,
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"content-length", str(len(TOO_LARGE_BODY)).encode()),
|
||||
],
|
||||
}
|
||||
)
|
||||
await send({"type": "http.response.body", "body": TOO_LARGE_BODY})
|
||||
|
||||
|
||||
class BodyLimitMiddleware:
|
||||
"""Refuses any request body over its route's ceiling, before the application sees it.
|
||||
|
||||
Registered just inside CORS in `main.py` — so the 413 is decorated like every other response
|
||||
and a browser can actually read it — but outside the session, the setup gate and every auth
|
||||
dependency, none of which run for a request that is already too large.
|
||||
|
||||
Every request gets a finite ceiling: `ROUTE_MAX_BODY_BYTES` raises it for the two routes that
|
||||
genuinely take bulk bytes, and there is no path that skips counting altogether.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app: ASGIApp,
|
||||
*,
|
||||
limit: int = DEFAULT_MAX_BODY_BYTES,
|
||||
routes: dict[tuple[str, str], int] | None = None,
|
||||
) -> None:
|
||||
self.app = app
|
||||
self.limit = limit
|
||||
self._routes = tuple(
|
||||
(method, template_to_regex(path), cap)
|
||||
for (method, path), cap in (ROUTE_MAX_BODY_BYTES if routes is None else routes).items()
|
||||
)
|
||||
|
||||
def limit_for(self, method: str, path: str) -> int:
|
||||
"""This request's ceiling: the route's own if it has one, else the global default. Every
|
||||
request gets a finite number — there is no path through here that is simply not counted."""
|
||||
for m, rx, cap in self._routes:
|
||||
if m == method and rx.match(path):
|
||||
return cap
|
||||
return self.limit
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
limit = self.limit_for(scope.get("method", ""), scope.get("path", ""))
|
||||
|
||||
declared = _declared_length(scope)
|
||||
if declared is not None and declared > limit:
|
||||
# Refused on the declaration alone: nothing downstream is called, and not one byte of
|
||||
# the body is read.
|
||||
await _send_too_large(send)
|
||||
return
|
||||
|
||||
seen = 0
|
||||
refused = False
|
||||
started = False
|
||||
|
||||
async def counting_receive() -> Message:
|
||||
nonlocal seen, refused
|
||||
message = await receive()
|
||||
if message["type"] != "http.request":
|
||||
return message
|
||||
seen += len(message.get("body", b""))
|
||||
if seen > limit:
|
||||
# The app is blocked on this receive, so it has not answered yet. Telling it the
|
||||
# client went away is the only in-band way to stop it reading: `request.stream()`
|
||||
# turns an `http.disconnect` into ClientDisconnect, which unwinds the handler. Our
|
||||
# own 413 goes out below, once the app has finished unwinding.
|
||||
refused = True
|
||||
return {"type": "http.disconnect"}
|
||||
return message
|
||||
|
||||
async def guarded_send(message: Message) -> None:
|
||||
nonlocal started
|
||||
if refused and not started:
|
||||
return # discard the app's answer to the disconnect; the 413 below is the response
|
||||
started = True
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, counting_receive, guarded_send)
|
||||
except Exception: # only swallowed when WE caused it, see below
|
||||
# ClientDisconnect (or whatever a handler raises on a truncated read) is the expected
|
||||
# consequence of the disconnect we just faked, and the caller must get the 413 rather
|
||||
# than a 500. Anything raised when we did NOT refuse is a real failure and is re-raised
|
||||
# untouched.
|
||||
if not refused:
|
||||
raise
|
||||
|
||||
if refused and not started:
|
||||
await _send_too_large(send)
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Alphabetical order in the reader's language.
|
||||
|
||||
Why not the database default: the Postgres image is Alpine, and musl's `en_US.utf8` collation is BYTE
|
||||
order — every accented initial sorted after "Z" ("Zorró < Életrajzi < Ötödik"), in every A→Z list the
|
||||
server orders. Postgres ships the ICU collations on the same image (dev, test, prod alike), so a text
|
||||
sort names one: Hungarian puts "Ö" after "O" and "Cs" after "Cu", English folds the accents in.
|
||||
|
||||
`collated` wraps an ORDER BY expression — and because a keyset cursor compares with the SAME expression
|
||||
(`feedquery.sort_keys`), the cursor pages in the same order it sorted in. `sorted_words` orders a Python
|
||||
list the same way through the database, so a list built in Python matches one built in SQL exactly.
|
||||
|
||||
The feed's title sorts ride indexes built in these collations (migration 0096), and an index keeps the
|
||||
order of the ICU that built it. `postgres:16-alpine` is a floating tag: an image update can bring a newer
|
||||
ICU, after which those indexes disagree with the comparisons a query makes (a keyset page skips or
|
||||
repeats a title). `heal_drift` rebuilds them at startup once Postgres reports the version moved."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Engine, SQLColumnExpression, column, func, select, text, values
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.types import String
|
||||
|
||||
from app import logs
|
||||
from app.mailtext import normalize
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
ICU: dict[str, str] = {"en": "en-x-icu", "hu": "hu-x-icu"}
|
||||
|
||||
|
||||
def collation(lang: str | None) -> str:
|
||||
"""The ICU collation for a UI language; anything unknown sorts as English."""
|
||||
return ICU[normalize(lang)]
|
||||
|
||||
|
||||
def collated(expr: SQLColumnExpression[Any], lang: str | None) -> ColumnElement[Any]:
|
||||
"""`expr` compared in the reader's alphabet. Case-folded, so "apple" and "Apple" sit together."""
|
||||
return func.lower(expr).collate(collation(lang))
|
||||
|
||||
|
||||
def sorted_words(db: Session, words: list[str], lang: str | None) -> list[str]:
|
||||
"""`words` in the reader's alphabetical order, by the same ICU collation as `collated`."""
|
||||
if not words:
|
||||
return []
|
||||
w = values(column("w", String), name="words").data([(x,) for x in words])
|
||||
return list(db.scalars(select(w.c.w).order_by(collated(w.c.w, lang), w.c.w)))
|
||||
|
||||
|
||||
def heal_drift(engine: Engine) -> list[str]:
|
||||
"""Rebuild every index on an ICU collation whose library version moved, then record the new version.
|
||||
Returns the rebuilt indexes. CONCURRENTLY: the feed keeps reading while it runs."""
|
||||
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
|
||||
drifted = conn.execute(
|
||||
text(
|
||||
"SELECT oid, collname FROM pg_collation WHERE collname = ANY(:names) "
|
||||
"AND collversion IS DISTINCT FROM pg_collation_actual_version(oid)"
|
||||
),
|
||||
{"names": list(ICU.values())},
|
||||
).all()
|
||||
if not drifted:
|
||||
return []
|
||||
indexes = list(
|
||||
conn.execute(
|
||||
text(
|
||||
"SELECT DISTINCT d.objid::regclass::text FROM pg_depend d "
|
||||
"JOIN pg_class c ON c.oid = d.objid AND c.relkind = 'i' "
|
||||
"WHERE d.classid = 'pg_class'::regclass "
|
||||
"AND d.refclassid = 'pg_collation'::regclass AND d.refobjid = ANY(:oids)"
|
||||
),
|
||||
{"oids": [row.oid for row in drifted]},
|
||||
).scalars()
|
||||
)
|
||||
log.warning(
|
||||
"ICU collation version changed (%s): rebuilding %d indexes",
|
||||
", ".join(row.collname for row in drifted),
|
||||
len(indexes),
|
||||
)
|
||||
for name in indexes:
|
||||
conn.execute(text(f"REINDEX INDEX CONCURRENTLY {name}"))
|
||||
for row in drifted:
|
||||
conn.execute(text(f'ALTER COLLATION "{row.collname}" REFRESH VERSION'))
|
||||
return indexes
|
||||
+38
-21
@@ -1,7 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args, get_origin
|
||||
|
||||
from pydantic import ValidationInfo, field_validator, model_validator
|
||||
from pydantic import AliasChoices, Field, ValidationInfo, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Shipped placeholder; production must override it. Used by the startup guard below.
|
||||
@@ -146,8 +146,6 @@ class Settings(BaseSettings):
|
||||
autotag_interval_minutes: int = 30
|
||||
subscriptions_resync_minutes: int = 360
|
||||
playlist_sync_minutes: int = 360
|
||||
# How often the shared demo account is auto-reset to a clean baseline (communal sandbox).
|
||||
demo_reset_minutes: int = 720
|
||||
# --- Maintenance / validation job ---
|
||||
# Runs once a day by default (admin-tunable via the Scheduler dashboard). Grace periods
|
||||
# are in days because that's the granularity that matters; the rolling re-validation
|
||||
@@ -173,9 +171,13 @@ class Settings(BaseSettings):
|
||||
feed_default_hidden_live: str = "live,upcoming"
|
||||
|
||||
# --- Download center (yt-dlp) ---
|
||||
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
|
||||
# Root directory (inside the container) for the files library — a Docker volume/bind mount.
|
||||
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
|
||||
download_root: str = "/downloads"
|
||||
# DOWNLOAD_ROOT is its old name: an install still on the old compose file (mounted at
|
||||
# /downloads, DOWNLOAD_ROOT=/downloads) keeps finding its files after it pulls a new image.
|
||||
files_root: str = Field(
|
||||
default="/files", validation_alias=AliasChoices("FILES_ROOT", "DOWNLOAD_ROOT", "files_root")
|
||||
)
|
||||
# Whether THIS process runs the download worker loop. On for the dedicated worker
|
||||
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
|
||||
worker_enabled: bool = False
|
||||
@@ -183,15 +185,18 @@ class Settings(BaseSettings):
|
||||
# for YouTube Proof-of-Origin tokens (bypasses bot-detection + unlocks high-quality
|
||||
# formats). Empty disables it (yt-dlp still works, but may hit "confirm you're not a bot").
|
||||
download_pot_base_url: str = "http://bgutil-pot:4416"
|
||||
# yt-dlp player clients (comma-separated). PRIMARY = android_vr: high-quality DASH, no PO
|
||||
# token / JS runtime needed, reliable for normal use. On a bot/DRM/format failure (YouTube's
|
||||
# per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
|
||||
# worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
|
||||
# Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
|
||||
# yt-dlp player clients (comma-separated). PRIMARY = tv_simply: measured 2026-10-10 as the one
|
||||
# client that is offered 1080p+ AND finishes the fetch, its stream token minted by the bgutil
|
||||
# sidecar. android_vr (the primary until then) was still offered 2160p, but every fetch died
|
||||
# with HTTP 403 — it now needs a stream PO token the sidecar cannot mint — and the fallback
|
||||
# below served 360p only, so every download came out at 360p without failing. On a
|
||||
# bot/DRM/format failure the worker retries with the FALLBACK set — the POT-capable web clients
|
||||
# (web needs the token + Deno) plus android, whose 360p is the last resort. Both admin-tunable
|
||||
# (env) when YouTube shifts what works.
|
||||
# A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients
|
||||
# of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there.
|
||||
# See adapters.YouTubeAdapter._COOKIE_CAPABLE.
|
||||
download_player_clients: str = "android_vr"
|
||||
download_player_clients: str = "tv_simply"
|
||||
download_player_clients_fallback: str = "web_safari,web,android"
|
||||
|
||||
@staticmethod
|
||||
@@ -244,10 +249,14 @@ class Settings(BaseSettings):
|
||||
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
|
||||
download_default_max_concurrent: int = 2
|
||||
download_default_max_jobs: int = 50
|
||||
# Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker
|
||||
# Per-file cap for local uploads. Unlike a download (whose size is unknown until the worker
|
||||
# finishes), an upload's size is known at ingest, so this bounds a single file directly; the
|
||||
# user's overall byte footprint is still governed by download_default_max_bytes.
|
||||
download_upload_max_bytes: int = 524_288_000 # 500 MiB
|
||||
# ge=1: this number is also the EDGE ceiling (`bodylimit.ROUTE_MAX_BODY_BYTES`), where 0 is a
|
||||
# real limit of zero rather than the route's "no per-file cap" sentinel — an instance set to 0
|
||||
# would refuse every upload, including the admin page's own way back. To lift the cap in
|
||||
# practice, set a large number.
|
||||
download_upload_max_bytes: int = Field(default=524_288_000, ge=1) # 500 MiB
|
||||
# Retention: an asset expires this many days after its last access; the GC job then
|
||||
# deletes it (once no job references it) after an additional grace window.
|
||||
download_retention_days: int = 30
|
||||
@@ -256,12 +265,15 @@ class Settings(BaseSettings):
|
||||
# Admin audit log: how long to keep rows (days; 0 = keep forever) + how often the GC runs.
|
||||
audit_retention_days: int = 180
|
||||
audit_gc_minutes: int = 1440
|
||||
# YouTube quota history (quota_events): days kept (0 = forever). A year, so the admin usage view's
|
||||
# longest window (365 days) stays whole; the same daily cleanup job prunes it.
|
||||
quota_retention_days: int = 365
|
||||
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
||||
download_layout: DownloadLayout = "plex"
|
||||
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
||||
sponsorblock_default: bool = False
|
||||
|
||||
# --- Diagnostics / logging (E22) ---
|
||||
# --- Diagnostics / logging ---
|
||||
# BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at
|
||||
# startup and on change without a restart. This is what a process logs at before it can reach
|
||||
# the database — and the only knob the worker/runner children read, since they are short-lived.
|
||||
@@ -294,7 +306,7 @@ class Settings(BaseSettings):
|
||||
plex_path_map: str = ""
|
||||
# Which Plex library section keys are exposed (comma-separated). Empty = all enabled sections.
|
||||
plex_libraries: str = ""
|
||||
# HU-overlay mapping (E5 S5): comma-separated `huSectionKey:enSectionKey` pairs. Each named HU
|
||||
# HU-overlay mapping: comma-separated `huSectionKey:enSectionKey` pairs. Each named HU
|
||||
# section is synced as a non-browsable overlay whose Hungarian metadata is written onto the EN
|
||||
# twin section's rows (matched by guid). Empty = no overlays. E.g. "3:4,5:1".
|
||||
plex_overlay_map: str = ""
|
||||
@@ -304,7 +316,7 @@ class Settings(BaseSettings):
|
||||
plex_sync_interval_min: int = 360
|
||||
# How often the incremental Plex→Siftlode watch-state reconcile runs, in minutes (Phase C):
|
||||
# a cheap history + on-deck pull plus a re-push of any states that failed their immediate push.
|
||||
# 15 min (S2): with Plex now the source of truth (prompt write-back on the ~10s checkpoint +
|
||||
# 15 min: with Plex now the source of truth (prompt write-back on the ~10s checkpoint +
|
||||
# Plex-authoritative pull), a faster incremental pull keeps Siftlode close to Plex-side changes.
|
||||
plex_watch_sync_interval_min: int = 15
|
||||
# How often the FULL watch-state reconcile runs, in minutes (Phase C). A whole-section rescan
|
||||
@@ -319,7 +331,7 @@ class Settings(BaseSettings):
|
||||
# each one is a whole segmented copy of the film on the scratch volume, written faster than
|
||||
# realtime. One account holds at most two regardless (plex/stream.py `_MAX_SESSIONS_PER_USER`).
|
||||
plex_max_sessions: int = 8
|
||||
# --- Transcode (playable="transcode") re-encode tuning (E6 S1) ---
|
||||
# --- Transcode (playable="transcode") re-encode tuning ---
|
||||
# Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the
|
||||
# source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render
|
||||
# node exists (skip the decodability gate); "off" = always software. See plex/stream.py `_select_encode`.
|
||||
@@ -337,12 +349,12 @@ class Settings(BaseSettings):
|
||||
plex_transcode_crf: int = 20
|
||||
plex_transcode_max_height: int = 1080
|
||||
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
|
||||
# dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point
|
||||
# this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast
|
||||
# dir under files_root. On localdev the files_root is a slow Windows bind-mount, so point
|
||||
# this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's files_root is fast
|
||||
# local disk, so the default is fine there.
|
||||
plex_hls_dir: str = ""
|
||||
|
||||
# --- TMDB enrichment (E5 S2; optional — enriches the Plex catalog from TMDB) ---
|
||||
# --- TMDB enrichment (optional — enriches the Plex catalog from TMDB) ---
|
||||
# TMDB API key (v3 api_key, 32 hex chars). Server-side only; stored encrypted when set via the
|
||||
# admin Config page. Empty → the TMDB enrichment job no-ops. Feeds keywords / crew / budget /
|
||||
# revenue / TMDb score / recommendations / backdrops onto the mirrored Plex rows, matched by the
|
||||
@@ -352,7 +364,7 @@ class Settings(BaseSettings):
|
||||
# Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default.
|
||||
tmdb_language: str = "en-US"
|
||||
|
||||
# --- Bug reports (E16): where a debug session's issue goes. All three empty = no tracker; the
|
||||
# --- Bug reports: where a debug session's issue goes. All three empty = no tracker; the
|
||||
# composer still exports a zip. The repository must be PRIVATE — an issue carries the reporter's
|
||||
# log — and app/issues/gitea.py refuses a public one. Runtime knobs (sysconfig, admin UI). ---
|
||||
issues_gitea_url: str = ""
|
||||
@@ -362,6 +374,11 @@ class Settings(BaseSettings):
|
||||
tmdb_enrich_interval_min: int = 120
|
||||
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||
tmdb_enrich_batch: int = 100
|
||||
# How often the people job fetches a batch of person bios from TMDB and portraits through Plex,
|
||||
# in minutes, and how many of each per run: ~18k linked people at 300 per 10 min is about ten hours
|
||||
# once, then only the newcomers.
|
||||
people_enrich_interval_min: int = 10
|
||||
people_enrich_batch: int = 300
|
||||
|
||||
@property
|
||||
def allowed_email_set(self) -> set[str]:
|
||||
|
||||
+51
-2
@@ -1,6 +1,6 @@
|
||||
from collections.abc import Iterator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
from app.config import settings
|
||||
@@ -12,7 +12,9 @@ class Base(DeclarativeBase):
|
||||
|
||||
# Pool sized above the sync-endpoint threadpool (~40) so a burst of short DB sessions can't starve on
|
||||
# checkout. The image proxies deliberately DON'T hold a connection during their slow external fetch
|
||||
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones.
|
||||
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones. The one
|
||||
# long hold — a playlist sync's advisory claim, for a whole push — opens its own unpooled connection
|
||||
# (sync/playlists.py `_claim_engine`) instead of drawing a third one from here.
|
||||
engine = create_engine(
|
||||
settings.database_url,
|
||||
pool_pre_ping=True,
|
||||
@@ -34,3 +36,50 @@ def get_db() -> Iterator[Session]:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def release_read_connection(db: Session) -> None:
|
||||
"""End a request's READ transaction so its pooled connection goes back BEFORE a long await.
|
||||
|
||||
SQLAlchemy 2.0 autobegins on the first SELECT, so a request that has merely authenticated is
|
||||
already holding one of the connections above — `idle in transaction` to Postgres, and nothing
|
||||
times it out. A route that then awaits something slow (a request body still arriving, an
|
||||
external fetch) pins it for the whole wait, which is precisely the invariant the pool is sized
|
||||
on in the comment above. Measured: the next checkout after this call reuses the very backend
|
||||
that was released, and a 1.8 s upload goes from one sustained `idle in transaction` to none.
|
||||
|
||||
Every route that reads its own body from a dependency needs this, and `test_body_limit.py`
|
||||
holds that — the three that do are the ones with a raised (or self-imposed) body ceiling, and
|
||||
they got here one at a time, which is how the upload route ended up the only one fixed.
|
||||
|
||||
The caller must have NOTHING PENDING, and that is checked rather than described. The session is
|
||||
shared with every dependency that ran before, so a rollback here would silently discard a write
|
||||
one of them staged — `service.enqueue` already learned that lesson the expensive way and uses a
|
||||
SAVEPOINT for it. CLAUDE.md: a rule stated in a comment must also be enforced in code.
|
||||
|
||||
TWO checks, because the obvious one is blind to the shapes this codebase actually writes in.
|
||||
`db.new`/`deleted`/`dirty` hold only UNFLUSHED unit-of-work: the moment a helper calls
|
||||
`db.flush()` — CLAUDE.md's canonical split is "the handler commits; helpers flush" — the object
|
||||
is persistent with no unflushed history and all three sets are empty again, and Core DML
|
||||
(`db.execute(update(...))`) never enters them at all. Measured: after `add`+`flush` the sets are
|
||||
empty, the old check passed, and the rollback lost the row. So the transaction is asked
|
||||
directly. Postgres assigns a transaction id on the first WRITE and never for a read, so
|
||||
`pg_current_xact_id_if_assigned()` is NULL for exactly the state this function is safe in — and
|
||||
it sees a raw `text("UPDATE …")` too, which no session-level bookkeeping would.
|
||||
"""
|
||||
if db.new or db.deleted or db.dirty:
|
||||
raise RuntimeError(
|
||||
"release_read_connection() with pending work: something ahead of this dependency "
|
||||
"staged a write, which ending the transaction would discard. Commit it first, or do "
|
||||
"it after the await."
|
||||
)
|
||||
if (
|
||||
db.in_transaction()
|
||||
and db.execute(text("SELECT pg_current_xact_id_if_assigned()")).scalar() is not None
|
||||
):
|
||||
raise RuntimeError(
|
||||
"release_read_connection() on a transaction that has already WRITTEN: something ahead "
|
||||
"of this dependency flushed a row or ran an UPDATE/INSERT/DELETE, which ending the "
|
||||
"transaction would discard. Commit it first, or do it after the await."
|
||||
)
|
||||
db.rollback()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, and the evidence a bug report is built from.
|
||||
"""Debug sessions: Start → reproduce → Stop, and the evidence a bug report is built from.
|
||||
|
||||
While a session is open its user's browser names it in `X-Debug-Session` on every request. The auth
|
||||
dependency accepts the header only for the caller's OWN open session (anything else is ignored, never
|
||||
@@ -269,7 +269,7 @@ def set_delivery(db: Session, session: DebugSession, state: dict, token: str) ->
|
||||
every step. It is stamped from `token` here instead, and only after the ROW confirms the claim
|
||||
is still ours. Without that check the write is a full-column overwrite with no WHERE: a send
|
||||
that lost its lease came back and stomped the take-over send's committed links, then put its
|
||||
own dead token back on an idle session (deep-review round 4). `release_send` has always made
|
||||
own dead token back on an idle session. `release_send` has always made
|
||||
the same check; this is the other half of it."""
|
||||
if claim_owner(db, session) != token:
|
||||
raise SendClaimLost(session.id)
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
"""Demo-account service: reset the shared demo sandbox to a clean, seeded baseline.
|
||||
|
||||
Shared by the admin's manual reset endpoint (routes/admin.py) and the scheduled demo-reset job
|
||||
(scheduler.py). Lives in the service layer, not routes, so the scheduler can call it without an
|
||||
`app.routes -> app.scheduler` import cycle (the reason its import used to be lazy).
|
||||
"""
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Playlist, PlaylistItem, Subscription, User, Video, VideoState
|
||||
|
||||
|
||||
def _seed_demo_playlists(db: Session, demo: User) -> int:
|
||||
"""Seed a few sample playlists from the shared catalog so a freshly-reset demo isn't
|
||||
empty — gives visitors something to play with. Best-effort: skips silently if the
|
||||
catalog is too small."""
|
||||
recent = (
|
||||
db.execute(
|
||||
select(Video.id)
|
||||
.where(Video.is_short.is_(False), Video.live_status == "none")
|
||||
.order_by(Video.published_at.desc().nullslast())
|
||||
.limit(60)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not recent:
|
||||
return 0
|
||||
samples = [
|
||||
("Fresh picks", recent[0:10]),
|
||||
("Quick watch", recent[10:18]),
|
||||
("Saved for later", recent[18:26]),
|
||||
]
|
||||
created = 0
|
||||
for pos, (name, vids) in enumerate(samples):
|
||||
if not vids:
|
||||
continue
|
||||
pl = Playlist(user_id=demo.id, name=name, position=pos)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for ipos, vid in enumerate(vids):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=ipos))
|
||||
created += 1
|
||||
return created
|
||||
|
||||
|
||||
def _seed_demo_subscriptions(db: Session, demo: User, n: int = 14) -> int:
|
||||
"""Seed the demo with a handful of catalog channels (the ones with the most videos) so its
|
||||
Channel manager isn't empty — gives visitors something to explore and matches the landing-page
|
||||
screenshot. Idempotent: clears the demo's subscriptions first, then re-adds the top N."""
|
||||
db.execute(delete(Subscription).where(Subscription.user_id == demo.id))
|
||||
top = (
|
||||
db.execute(
|
||||
select(Video.channel_id)
|
||||
.where(Video.channel_id.is_not(None))
|
||||
.group_by(Video.channel_id)
|
||||
.order_by(func.count().desc())
|
||||
.limit(n)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for channel_id in top:
|
||||
db.add(Subscription(user_id=demo.id, channel_id=channel_id))
|
||||
return len(top)
|
||||
|
||||
|
||||
def reset_demo_state(db: Session, demo: User) -> int:
|
||||
"""Wipe the demo account's per-user state (watch/save/hide states, playlists, preferences)
|
||||
back to a clean baseline and re-seed a few sample playlists + channel subscriptions. Returns
|
||||
the seeded playlist count. Shared by the admin's manual reset button and the scheduled demo
|
||||
reset (see scheduler)."""
|
||||
db.execute(delete(VideoState).where(VideoState.user_id == demo.id))
|
||||
# Playlist items cascade on playlist delete (ondelete="CASCADE").
|
||||
db.execute(delete(Playlist).where(Playlist.user_id == demo.id))
|
||||
demo.preferences = {"language": "en"}
|
||||
db.commit()
|
||||
seeded = _seed_demo_playlists(db, demo)
|
||||
_seed_demo_subscriptions(db, demo)
|
||||
db.commit()
|
||||
return seeded
|
||||
@@ -1,4 +1,4 @@
|
||||
"""A user's diagnostics level (E22 S5): what they consented to, and for how long.
|
||||
"""A user's diagnostics level: what they consented to, and for how long.
|
||||
|
||||
Three levels, named in the UI Alap / Részletes / Teljes:
|
||||
|
||||
@@ -10,7 +10,6 @@ Three levels, named in the UI Alap / Részletes / Teljes:
|
||||
|
||||
A raised level always lapses after `ELEVATION_HOURS`: the expiry is computed HERE at consent time,
|
||||
never taken from the client, and an expired level reads as INFO without anything having to clear it.
|
||||
The shared demo account cannot raise its level — its log would mix every visitor's session.
|
||||
|
||||
Core layer: imports `models` and `sysconfig`.
|
||||
"""
|
||||
@@ -27,7 +26,7 @@ DiagnosticsLevel = Literal["INFO", "DEBUG", "TRACE"]
|
||||
LEVELS: tuple[str, ...] = get_args(DiagnosticsLevel)
|
||||
BASE: DiagnosticsLevel = "INFO"
|
||||
ELEVATION_HOURS = 24
|
||||
# How long a debug session records at most (E16, app/debugsession.py). Here, beside the other
|
||||
# How long a debug session records at most (app/debugsession.py). Here, beside the other
|
||||
# consent window, because the start dialog's consent text states it and reads it from `state`.
|
||||
SESSION_MINUTES = 60
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""E24 — the read-model behind the Home surface ("Mit nézzek most?").
|
||||
"""The read-model behind the Home surface ("Mit nézzek most?").
|
||||
|
||||
The anti-paralysis answer layer: one hero suggestion, a cross-source resume rail, and the user's own
|
||||
comfort shelf, all served from our own Postgres with no upstream call at serve time (the same rule
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The time budget — E24's replacement for "what mood are you in?".
|
||||
"""The time budget — the replacement for "what mood are you in?".
|
||||
|
||||
`random-gondolatok` 15/D asked for a mood questionnaire before the recommendations. The same file's
|
||||
later BA-interview note (#23) says the mood half is not what makes this work: the friction is
|
||||
|
||||
@@ -48,6 +48,7 @@ def pick(
|
||||
shelf: list[dict] | None = None,
|
||||
exclude: set[str] | None = None,
|
||||
start: int = 0,
|
||||
lang: str,
|
||||
) -> dict | None:
|
||||
"""`{"tier": ..., "card": ...}`, or None when nothing at all fits.
|
||||
|
||||
@@ -70,8 +71,8 @@ def pick(
|
||||
skip = exclude or set()
|
||||
|
||||
def _resume() -> dict | None:
|
||||
for card in rails.continue_watching(db, user) if resume is None else resume:
|
||||
playable = rails.playable(db, user.id, card)
|
||||
for card in rails.continue_watching(db, user, lang=lang) if resume is None else resume:
|
||||
playable = rails.playable(db, user.id, card, lang=lang)
|
||||
# Excluded on the card the USER saw (the show), not only on what it resolves to, so a
|
||||
# rerolled-past show does not come straight back as its own next episode.
|
||||
if cards.key(card) in skip or (playable is not None and cards.key(playable) in skip):
|
||||
@@ -82,7 +83,7 @@ def pick(
|
||||
|
||||
def _comfort() -> dict | None:
|
||||
if shelf is None:
|
||||
own = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
||||
own = pins.shelf(db, user, include_plex=rails.plex_available(db), lang=lang)
|
||||
else:
|
||||
own = shelf
|
||||
for card in pins.playables(own):
|
||||
@@ -93,7 +94,7 @@ def pick(
|
||||
def _surprise() -> dict | None:
|
||||
# A wider draw than the one card we need: the shuffle is seeded, so without the headroom an
|
||||
# excluded pick would leave the tier empty rather than move past it.
|
||||
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW):
|
||||
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW, lang=lang):
|
||||
if cards.key(card) not in skip:
|
||||
return card
|
||||
return None
|
||||
|
||||
@@ -24,7 +24,7 @@ from sqlalchemy.orm import Session
|
||||
from app.discover import cards, queries
|
||||
from app.models import Channel, ComfortPin, PlexItem, PlexShow, Playlist, User, Video
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, show_card
|
||||
from app.plex.cards import SHOW_NEW, leaf_card, show_card
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
# kind -> the ComfortPin column that holds it. The client's vocabulary and the schema's, in one place.
|
||||
@@ -46,8 +46,8 @@ def resolve_ref(db: Session, user: User, kind: str, ref: str) -> tuple[str, obje
|
||||
translation belongs here rather than in five call sites.
|
||||
|
||||
The COLUMN half of every answer comes from `PIN_COLUMNS`; only the LOOKUP differs per kind
|
||||
(a direct `db.get`, a rating-key translation, an int parse plus an ownership check). Before the
|
||||
E24 review the branches spelled the column names out again and `PIN_COLUMNS` had no readers at
|
||||
(a direct `db.get`, a rating-key translation, an int parse plus an ownership check). Before a
|
||||
review the branches spelled the column names out again and `PIN_COLUMNS` had no readers at
|
||||
all — a constant that called itself the one vocabulary while nothing spoke it."""
|
||||
col = PIN_COLUMNS.get(kind)
|
||||
if col is None:
|
||||
@@ -170,7 +170,7 @@ class _Batch:
|
||||
return b
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict | None:
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch, lang: str) -> dict | None:
|
||||
"""One pin as a unified card with its resolved `next`. None when the pinned row has gone (the FK
|
||||
cascade removes the pin itself, so this is only the window between a delete and a refetch)."""
|
||||
card: dict | None = None
|
||||
@@ -196,16 +196,16 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict
|
||||
if it is None:
|
||||
return None
|
||||
st = rollup.item_state_map(db, user.id, [it.id]).get(it.id)
|
||||
card = cards.from_plex(leaf_card(it, st))
|
||||
card = cards.from_plex(leaf_card(it, st, lang=lang))
|
||||
nxt = dict(card) # a movie or a single episode plays itself
|
||||
|
||||
elif pin.plex_show_id is not None:
|
||||
sh = db.get(PlexShow, pin.plex_show_id)
|
||||
if sh is None:
|
||||
return None
|
||||
status = rollup.state_map(db, user.id, [sh.id]).get(sh.id, "new")
|
||||
card = cards.from_plex(show_card(sh, status))
|
||||
ep = queries.next_episode(db, user.id, sh.id)
|
||||
state = rollup.state_map(db, user.id, [sh.id]).get(sh.id, SHOW_NEW)
|
||||
card = cards.from_plex(show_card(sh, state, lang=lang))
|
||||
ep = queries.next_episode(db, user.id, sh.id, lang=lang)
|
||||
nxt = cards.from_plex(ep) if ep else None
|
||||
|
||||
elif pin.playlist_id is not None:
|
||||
@@ -229,7 +229,7 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict
|
||||
_PLEX_COLUMNS = ("plex_item_id", "plex_show_id")
|
||||
|
||||
|
||||
def shelf(db: Session, user: User, *, include_plex: bool = True) -> list[dict]:
|
||||
def shelf(db: Session, user: User, *, include_plex: bool = True, lang: str) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order.
|
||||
|
||||
`include_plex=False` is the Home surface with the deployment's Plex toggle off (the owner's call,
|
||||
@@ -243,7 +243,7 @@ def shelf(db: Session, user: User, *, include_plex: bool = True) -> list[dict]:
|
||||
pins = [p for p in pins if all(getattr(p, c) is None for c in _PLEX_COLUMNS)]
|
||||
batch = _Batch.load(db, user, pins)
|
||||
for pin in pins:
|
||||
card = _shelf_card(db, user, pin, batch)
|
||||
card = _shelf_card(db, user, pin, batch, lang)
|
||||
if card is not None:
|
||||
out.append(card)
|
||||
return out
|
||||
|
||||
@@ -40,7 +40,7 @@ from app.models import (
|
||||
VideoState,
|
||||
)
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, movie_card, show_card
|
||||
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||
from app.plex.query import apply_show_state_filter, content_libs
|
||||
|
||||
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
|
||||
@@ -221,7 +221,9 @@ def _unseen_movie_conds(uid: int):
|
||||
return ~seen
|
||||
|
||||
|
||||
def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tuple[object, dict]]:
|
||||
def plex_in_progress(
|
||||
db: Session, uid: int, limit: int = RAIL_LIMIT, *, lang: str
|
||||
) -> list[tuple[object, dict]]:
|
||||
"""In-progress movies + partially-watched shows as (progressed_at, card).
|
||||
|
||||
Mirrors `routes/plex/discover.py::_continue_watching` — same `PlexState` predicate for movies (a
|
||||
@@ -249,26 +251,32 @@ def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tup
|
||||
.limit(limit)
|
||||
)
|
||||
for it, st in db.execute(mq).all():
|
||||
out.append((st.progress_updated_at, movie_card(it, st)))
|
||||
out.append((st.progress_updated_at, movie_card(it, st, lang=lang)))
|
||||
|
||||
if show_libs:
|
||||
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
||||
sq = apply_show_state_filter(
|
||||
select(PlexShow, prog.c.resumed_at).join(prog, prog.c.show_id == PlexShow.id),
|
||||
select(PlexShow, prog.c.resumed_at, prog.c.total, prog.c.watched, prog.c.inprog).join(
|
||||
prog, prog.c.show_id == PlexShow.id
|
||||
),
|
||||
prog.c.total,
|
||||
prog.c.watched,
|
||||
prog.c.inprog,
|
||||
"in_progress",
|
||||
).where(PlexShow.library_id.in_(show_libs))
|
||||
for sh, resumed_at in db.execute(
|
||||
for sh, resumed_at, total, watched, inprog in db.execute(
|
||||
sq.order_by(prog.c.resumed_at.desc().nullslast()).limit(limit)
|
||||
).all():
|
||||
out.append((resumed_at, show_card(sh, "in_progress")))
|
||||
out.append(
|
||||
(resumed_at, show_card(sh, rollup.show_state(total, watched, inprog), lang=lang))
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) -> list[dict]:
|
||||
def plex_surprise(
|
||||
db: Session, uid: int, budget: str, seed: int, limit: int, *, lang: str
|
||||
) -> list[dict]:
|
||||
"""Unseen playable movies inside the budget, in seeded-shuffle order.
|
||||
|
||||
Movies only. A show is not a thing you can start playing, and resolving every candidate show to
|
||||
@@ -300,14 +308,14 @@ def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) ->
|
||||
q = q.where(PlexItem.duration_s <= hi)
|
||||
items = db.execute(q).scalars().all()
|
||||
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||
return [movie_card(it, states.get(it.id)) for it in items]
|
||||
return [movie_card(it, states.get(it.id), lang=lang) for it in items]
|
||||
|
||||
|
||||
# Watch states that take an episode out of the running for "what would play next".
|
||||
_DONE = ("watched", "hidden")
|
||||
|
||||
|
||||
def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
||||
def next_episode(db: Session, uid: int, show_id: int, *, lang: str) -> dict | None:
|
||||
"""What "put that show on" means: the episode in progress, else the first unwatched one, else the
|
||||
first — over the same `(season, episode)` broadcast ordering `detail.py::_episode_nav` uses.
|
||||
|
||||
@@ -357,5 +365,9 @@ def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
||||
ep, st = pick
|
||||
show = db.get(PlexShow, show_id)
|
||||
return leaf_card(
|
||||
ep, st, show_title=show.title if show else None, show_rk=show.rating_key if show else None
|
||||
ep,
|
||||
st,
|
||||
show_title=display_title(show, lang) if show else None,
|
||||
show_rk=show.rating_key if show else None,
|
||||
lang=lang,
|
||||
)
|
||||
|
||||
@@ -22,7 +22,7 @@ def plex_available(db: Session) -> bool:
|
||||
return sysconfig.get_bool(db, "plex_enabled")
|
||||
|
||||
|
||||
def playable(db: Session, uid: int, card: dict) -> dict | None:
|
||||
def playable(db: Session, uid: int, card: dict, *, lang: str) -> dict | None:
|
||||
"""The thing that would actually START from this card — itself for anything with a runtime, and
|
||||
for a Plex SHOW the episode that is next (in progress, else the first unwatched, else the first).
|
||||
|
||||
@@ -40,11 +40,11 @@ def playable(db: Session, uid: int, card: dict) -> dict | None:
|
||||
sh = by_rating_key(db, PlexShow, card["id"])
|
||||
if sh is None:
|
||||
return None
|
||||
ep = queries.next_episode(db, uid, sh.id)
|
||||
ep = queries.next_episode(db, uid, sh.id, lang=lang)
|
||||
return cards.from_plex(ep) if ep else None
|
||||
|
||||
|
||||
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[dict]:
|
||||
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT, *, lang: str) -> list[dict]:
|
||||
"""One resume rail across both sources, most recently progressed first.
|
||||
|
||||
A Plex SHOW enters as a show CARD — that is what the Plex rail shows, and it is the artwork the
|
||||
@@ -57,17 +57,22 @@ def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[
|
||||
(ts, cards.from_video(r)) for ts, r in queries.yt_in_progress(db, user, limit)
|
||||
]
|
||||
if plex_available(db):
|
||||
rows += [(ts, cards.from_plex(c)) for ts, c in queries.plex_in_progress(db, user.id, limit)]
|
||||
rows += [
|
||||
(ts, cards.from_plex(c))
|
||||
for ts, c in queries.plex_in_progress(db, user.id, limit, lang=lang)
|
||||
]
|
||||
rows.sort(key=lambda r: r[0] or NO_TS, reverse=True)
|
||||
out = [card for _, card in rows[:limit]]
|
||||
# Resolve AFTER the slice: only the rows that survive it are worth a query.
|
||||
for card in out:
|
||||
if card["kind"] == "show":
|
||||
card["next"] = playable(db, user.id, card)
|
||||
card["next"] = playable(db, user.id, card, lang=lang)
|
||||
return out
|
||||
|
||||
|
||||
def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) -> list[dict]:
|
||||
def surprise(
|
||||
db: Session, user: User, budget: str, seed: int, limit: int = 3, *, lang: str
|
||||
) -> list[dict]:
|
||||
"""The hero's draw pool: unseen things that fit the budget, from both sources under ONE seed.
|
||||
|
||||
Interleaved rather than concatenated — the same alternation `routes/plex/discover.py` uses for its
|
||||
@@ -83,7 +88,9 @@ def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) ->
|
||||
(cards.from_video, queries.yt_surprise(db, user, budget, seed, limit)),
|
||||
(
|
||||
cards.from_plex,
|
||||
queries.plex_surprise(db, user.id, budget, seed, limit) if plex_available(db) else [],
|
||||
queries.plex_surprise(db, user.id, budget, seed, limit, lang=lang)
|
||||
if plex_available(db)
|
||||
else [],
|
||||
),
|
||||
]
|
||||
if seed % 2:
|
||||
|
||||
@@ -106,7 +106,8 @@ class YouTubeAdapter(SourceAdapter):
|
||||
# `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
|
||||
# cookies`, drops it, and the request that is left with no client fails with "Requested format is
|
||||
# not available" — which is already a retriable marker above, so the cost is a wasted attempt
|
||||
# rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every
|
||||
# rather than a hard failure. Our primary set is tv_simply ALONE, which cannot carry cookies either
|
||||
# (measured on yt-dlp 2026.08.19: `SUPPORTS_COOKIES` False), so without this filter every
|
||||
# signed-in download would burn its first attempt that way before reaching the POT-backed web
|
||||
# fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
|
||||
# ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
|
||||
@@ -156,7 +157,7 @@ class YouTubeAdapter(SourceAdapter):
|
||||
|
||||
class FacebookAdapter(SourceAdapter):
|
||||
key = "facebook"
|
||||
# Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||||
# Measured (yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||||
# with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a
|
||||
# SECOND attempt behind that one error, not the default: the links that already download with the
|
||||
# cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without
|
||||
@@ -166,7 +167,7 @@ class FacebookAdapter(SourceAdapter):
|
||||
_POST_PAGE_MAX_BYTES = 16 * 1024 * 1024
|
||||
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
# Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
|
||||
# Measured: given a group post's link, yt-dlp either finds no formats or — worse —
|
||||
# downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
|
||||
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
|
||||
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
|
||||
@@ -176,7 +177,7 @@ class FacebookAdapter(SourceAdapter):
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
|
||||
page = raw.decode("utf-8", "replace")
|
||||
# Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
|
||||
# Measured: a refused or expired cookie gets the login page (~450 KB, a login
|
||||
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
|
||||
if _quoted(post_id) not in page:
|
||||
raise StatedFailure(POST_NOT_SHOWN_MESSAGE)
|
||||
|
||||
@@ -89,7 +89,7 @@ def run() -> dict:
|
||||
if a.poster_path is None:
|
||||
# Record a self-hosted poster for every download (the reliable og:image source):
|
||||
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
||||
rel = editmod.ensure_poster(a, settings.download_root)
|
||||
rel = editmod.ensure_poster(a, settings.files_root)
|
||||
if rel:
|
||||
a.poster_path = rel
|
||||
filled_poster += 1
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
"""Ad-hoc clips: a marked range of a YouTube video or a Plex item, cut into a new per-user asset.
|
||||
|
||||
A clip never needs the whole source first. A YouTube clip fetches only its range (yt-dlp's
|
||||
`download_ranges`; measured: 3-7 s whether the video runs four minutes or an hour, and the fetched
|
||||
range starts on the marked frame), and a Plex clip reads the library file in place. Either way the
|
||||
range then goes through ONE re-encode built here, so every clip has the same shape whatever its
|
||||
source: frame-exact at both ends, H.264 at most 1080p (an HDR source tonemapped to SDR — kept as
|
||||
10-bit PQ it played washed out), stereo AAC, in an .mp4 that plays on any phone.
|
||||
|
||||
A clip may also crop the picture to a frame the user drew over the playing video. The frame is
|
||||
given in fractions of the box it was drawn in, with that box's aspect: the YouTube iframe never
|
||||
tells its picture's size, so the frame there is drawn over the player's 16:9 box (a short's 9:16),
|
||||
and the cut maps it onto the real picture (`crop_filter`) — what lies over a letterbox bar is dropped.
|
||||
|
||||
The spec is stored on the job (`DownloadJob.edit_spec`) and is the clip's whole identity: its
|
||||
signature is the asset's cache key, so marking the same range again is an instant cache hit, and a
|
||||
different range or audio track is a different clip.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from app.downloads.edit import AAC_ARGS, EVEN_FRAME, INPUT_GUARD, X264_ARGS
|
||||
from app.plex.sync import HDR_TRANSFERS
|
||||
|
||||
SOURCES = ("youtube", "plex")
|
||||
# Owner's cap: a clip is a moment, not a way to export a whole film to a public link.
|
||||
MAX_CLIP_S = 600
|
||||
MIN_CLIP_S = 1
|
||||
MAX_HEIGHT = 1080
|
||||
# What yt-dlp fetches the range in (formats.normalize fills the rest): the clip's own ceiling, so the
|
||||
# re-encode never upscales and never throws away a better source than it keeps.
|
||||
YOUTUBE_FETCH_SPEC = {"mode": "av", "max_height": MAX_HEIGHT, "container": "mp4"}
|
||||
# Bump when the spec → output-bytes mapping changes, so a cached clip re-derives.
|
||||
_SIG_VERSION = 1
|
||||
|
||||
_YOUTUBE_ID = re.compile(r"[A-Za-z0-9_-]{11}")
|
||||
_RATING_KEY = re.compile(r"[0-9]{1,32}")
|
||||
_MAX_AUDIO_TRACK = 63
|
||||
# HDR → SDR as measured on the owner's 4K HDR files (2026-10-10): linearise, tonemap (hable),
|
||||
# convert the primaries and transfer to BT.709 and back to 8-bit 4:2:0.
|
||||
_TONEMAP = (
|
||||
"zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,"
|
||||
"zscale=t=bt709:m=bt709:r=tv,format=yuv420p"
|
||||
)
|
||||
_SCALE = f"scale=-2:'min({MAX_HEIGHT},ih)'"
|
||||
# A crop frame is at least this share of its box on each side (the player's frame holds the same
|
||||
# floor), and what is left of it on the real picture at least _MIN_VISIBLE of it.
|
||||
MIN_CROP = 0.1
|
||||
_MIN_VISIBLE = 0.05
|
||||
# The box aspects a frame may be drawn in: a 9:16 short to a 21:9 film, with room either side.
|
||||
_CROP_ASPECTS = (0.25, 4.0)
|
||||
# Float slack on the frame's far edges (x + w, y + h), which the client rounds to 4 decimals.
|
||||
_EDGE_SLACK = 1e-3
|
||||
|
||||
|
||||
class ClipError(Exception):
|
||||
"""A clip request that cannot be cut; `reason` is an i18n-mappable key, `params` the numbers its
|
||||
message names (the cap itself, so the client never says a stale one)."""
|
||||
|
||||
def __init__(self, reason: str, **params: float):
|
||||
self.reason = reason
|
||||
self.params = params
|
||||
super().__init__(reason)
|
||||
|
||||
|
||||
def normalize(
|
||||
source: str,
|
||||
ref: str,
|
||||
start_s: float,
|
||||
end_s: float,
|
||||
audio_track: int | None = None,
|
||||
crop: dict | None = None,
|
||||
) -> dict:
|
||||
"""The validated, canonical clip spec. Times are rounded to the millisecond and a crop to four
|
||||
decimals, so the same marks always give the same cache key. Raises ClipError."""
|
||||
if source not in SOURCES:
|
||||
raise ClipError("bad_source")
|
||||
pattern = _YOUTUBE_ID if source == "youtube" else _RATING_KEY
|
||||
if not isinstance(ref, str) or not pattern.fullmatch(ref):
|
||||
raise ClipError("bad_source")
|
||||
if not (math.isfinite(start_s) and math.isfinite(end_s)) or start_s < 0:
|
||||
raise ClipError("bad_range")
|
||||
start, end = round(start_s, 3), round(end_s, 3)
|
||||
if end - start < MIN_CLIP_S:
|
||||
raise ClipError("too_short")
|
||||
if end - start > MAX_CLIP_S:
|
||||
raise ClipError("too_long", limit_s=MAX_CLIP_S)
|
||||
spec: dict = {"source": source, "ref": ref, "start_s": start, "end_s": end}
|
||||
if source == "plex" and audio_track is not None:
|
||||
if not 0 <= audio_track <= _MAX_AUDIO_TRACK:
|
||||
raise ClipError("bad_audio")
|
||||
spec["audio_track"] = audio_track
|
||||
frame = _crop(crop)
|
||||
if frame is not None:
|
||||
spec["crop"] = frame
|
||||
return spec
|
||||
|
||||
|
||||
def _crop(raw: dict | None) -> dict | None:
|
||||
"""A crop frame, canonical — or None for no crop, which a frame over the whole box also is (the
|
||||
same clip as one cut without a frame, so the same cache key)."""
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
x, y, w, h, aspect = (float(raw[k]) for k in ("x", "y", "w", "h", "aspect"))
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise ClipError("bad_crop") from e
|
||||
if not all(math.isfinite(v) for v in (x, y, w, h, aspect)):
|
||||
raise ClipError("bad_crop")
|
||||
lo, hi = _CROP_ASPECTS
|
||||
if not lo <= aspect <= hi or x < 0 or y < 0:
|
||||
raise ClipError("bad_crop")
|
||||
if w < MIN_CROP - _EDGE_SLACK or h < MIN_CROP - _EDGE_SLACK:
|
||||
raise ClipError("bad_crop")
|
||||
if x + w > 1 + _EDGE_SLACK or y + h > 1 + _EDGE_SLACK:
|
||||
raise ClipError("bad_crop")
|
||||
frame = {"x": x, "y": y, "w": min(w, 1 - x), "h": min(h, 1 - y), "aspect": aspect}
|
||||
frame = {k: round(v, 4) for k, v in frame.items()}
|
||||
if (frame["x"], frame["y"], frame["w"], frame["h"]) == (0, 0, 1, 1):
|
||||
return None
|
||||
return frame
|
||||
|
||||
|
||||
def duration(spec: dict) -> float:
|
||||
return spec["end_s"] - spec["start_s"]
|
||||
|
||||
|
||||
def sig(spec: dict) -> str:
|
||||
"""The clip's cache signature: the spec itself, versioned."""
|
||||
body = json.dumps({**spec, "_v": _SIG_VERSION}, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(body.encode()).hexdigest()[:24]
|
||||
|
||||
|
||||
def asset_ref(user_id: int, spec: dict) -> str:
|
||||
"""The asset's `source_ref`. The user id makes a clip private to whoever cut it, like an edit:
|
||||
never dedup-shared, never cache-served to someone else."""
|
||||
return f"{user_id}:{spec['source']}:{spec['ref']}"
|
||||
|
||||
|
||||
def reference_url(spec: dict) -> str | None:
|
||||
"""Where a YouTube clip came from: its video at the clip's start. A Plex clip has no public page."""
|
||||
if spec.get("source") != "youtube" or not spec.get("ref"):
|
||||
return None
|
||||
return f"https://www.youtube.com/watch?v={spec['ref']}&t={int(spec.get('start_s') or 0)}s"
|
||||
|
||||
|
||||
def is_hdr(probe: dict) -> bool:
|
||||
"""Whether the first video stream of an ffprobe `-show_streams` JSON carries an HDR transfer."""
|
||||
for s in probe.get("streams") or []:
|
||||
if s.get("codec_type") == "video":
|
||||
# PQ / HLG: an 8-bit SDR encode must tonemap them (the one set the Plex stream uses).
|
||||
return s.get("color_transfer") in HDR_TRANSFERS
|
||||
return False
|
||||
|
||||
|
||||
def display_aspect(probe: dict) -> float | None:
|
||||
"""The picture's shape as a player shows it, from an ffprobe `-show_streams` JSON: the first video
|
||||
stream's width × its sample aspect ratio over its height, turned a quarter when the stream says
|
||||
it is rotated (ffmpeg turns such a stream upright before any filter of ours, so a crop is cut
|
||||
from the upright picture). None when the stream has no size."""
|
||||
for s in probe.get("streams") or []:
|
||||
if s.get("codec_type") != "video":
|
||||
continue
|
||||
w, h = s.get("width"), s.get("height")
|
||||
if not isinstance(w, int) or not isinstance(h, int) or w <= 0 or h <= 0:
|
||||
return None
|
||||
dar = w * (_ratio(s.get("sample_aspect_ratio")) or 1.0) / h
|
||||
return 1 / dar if _quarter_turned(s) else dar
|
||||
return None
|
||||
|
||||
|
||||
def _ratio(v: object) -> float | None:
|
||||
"""'32:27' → 1.185…; '0:1', 'N/A', a missing or malformed value → None."""
|
||||
if not isinstance(v, str) or ":" not in v:
|
||||
return None
|
||||
num, _, den = v.partition(":")
|
||||
try:
|
||||
n, d = float(num), float(den)
|
||||
except ValueError: # silent-ok: a malformed ratio reads as none given; the caller takes 1:1
|
||||
return None
|
||||
return n / d if n > 0 and d > 0 else None
|
||||
|
||||
|
||||
def _quarter_turned(stream: dict) -> bool:
|
||||
rotation: object = (stream.get("tags") or {}).get("rotate")
|
||||
for sd in stream.get("side_data_list") or []:
|
||||
if "rotation" in sd:
|
||||
rotation = sd["rotation"]
|
||||
try:
|
||||
return round(float(str(rotation))) % 180 == 90
|
||||
except ValueError: # silent-ok: no (None) or a malformed rotation: the stream is upright
|
||||
return False
|
||||
|
||||
|
||||
def crop_filter(crop: dict, dar: float) -> str | None:
|
||||
"""The ffmpeg `crop` for a frame drawn in a box of `crop["aspect"]`, over a picture of display
|
||||
aspect `dar` fitted into that box as a player fits it (whole, centred, bars on the short sides).
|
||||
The frame's part over a bar is dropped: "" when what is left is the whole picture, None when too
|
||||
little of the frame lies on the picture at all.
|
||||
|
||||
In fractions of the input's own width and height, so a sample aspect ratio other than 1:1 (an
|
||||
anamorphic DVD) needs nothing of its own, and floored to even pixels: an even 4:2:0 frame, and a
|
||||
window that never runs past the picture's far edge."""
|
||||
box = crop["aspect"]
|
||||
pic_w, pic_h = (1.0, box / dar) if dar > box else (dar / box, 1.0)
|
||||
|
||||
def span(start: float, size: float, extent: float) -> tuple[float, float]:
|
||||
origin = (1 - extent) / 2
|
||||
a = min(1.0, max(0.0, (start - origin) / extent))
|
||||
b = min(1.0, max(0.0, (start + size - origin) / extent))
|
||||
return a, b - a
|
||||
|
||||
x, w = span(crop["x"], crop["w"], pic_w)
|
||||
y, h = span(crop["y"], crop["h"], pic_h)
|
||||
if w < _MIN_VISIBLE or h < _MIN_VISIBLE:
|
||||
return None
|
||||
if w > 1 - 1e-4 and h > 1 - 1e-4:
|
||||
return ""
|
||||
|
||||
def px(axis: str, f: float) -> str:
|
||||
return f"trunc({axis}*{f:.6f}/2)*2"
|
||||
|
||||
return f"crop={px('iw', w)}:{px('ih', h)}:{px('iw', x)}:{px('ih', y)}"
|
||||
|
||||
|
||||
def audio_track_count(probe: dict) -> int:
|
||||
return sum(1 for s in probe.get("streams") or [] if s.get("codec_type") == "audio")
|
||||
|
||||
|
||||
def build_cmd(
|
||||
src: Path,
|
||||
dest: Path,
|
||||
start_s: float,
|
||||
dur_s: float,
|
||||
*,
|
||||
audio_track: int | None,
|
||||
hdr: bool,
|
||||
threads: int,
|
||||
crop: str = "",
|
||||
) -> list[str]:
|
||||
"""ffmpeg for one clip: an input seek (decodes from the keyframe before `start_s` and drops the
|
||||
frames up to it, so the clip starts on the marked frame), the first video stream, the chosen audio
|
||||
track (the first when none was chosen; none at all is fine), no subtitles, data or chapters.
|
||||
A `crop` (crop_filter) comes first: the tonemap and the scale then work on the frame it keeps."""
|
||||
vf = ",".join([*([crop] if crop else []), *([_TONEMAP] if hdr else []), _SCALE, EVEN_FRAME])
|
||||
audio = f"0:a:{audio_track}" if audio_track is not None else "0:a:0?"
|
||||
return [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-nostats",
|
||||
"-ss",
|
||||
f"{start_s:.3f}",
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-t",
|
||||
f"{dur_s:.3f}",
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
audio,
|
||||
"-sn",
|
||||
"-dn",
|
||||
# A source's chapters would ride into the .mp4 as a data track of their own (an mkv's
|
||||
# chapter list came out as `bin_data`); a clip of a moment has no chapters to keep.
|
||||
"-map_chapters",
|
||||
"-1",
|
||||
"-vf",
|
||||
vf,
|
||||
*X264_ARGS,
|
||||
"-threads",
|
||||
str(threads),
|
||||
*AAC_ARGS,
|
||||
"-ac",
|
||||
"2",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
str(dest),
|
||||
]
|
||||
@@ -30,6 +30,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential, MediaAsset
|
||||
@@ -249,8 +250,8 @@ def usable_cookie(
|
||||
|
||||
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
||||
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
||||
InvalidToken) also yields None; the download then fails honestly on the login wall (S3's
|
||||
`login_required`)."""
|
||||
InvalidToken) also yields None; the download then fails honestly on the login wall (the
|
||||
`login_required` category)."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return None
|
||||
@@ -259,6 +260,28 @@ def usable_cookie(
|
||||
return None
|
||||
if not is_authed_asset(asset, spec, user_id, url):
|
||||
return None # shared asset — never attach a cookie to a non-private identity
|
||||
return _readable(cred, user_id, host_key)
|
||||
|
||||
|
||||
def usable_clip_cookie(
|
||||
db: Session, user_id: int, url: str | None, asset: MediaAsset, clip_spec: dict
|
||||
) -> str | None:
|
||||
"""`usable_cookie` for an ad-hoc clip's range fetch. The second gate is the clip's own identity:
|
||||
a clip asset is private by construction (`clip.asset_ref` names its user, see downloads/clip),
|
||||
so a cookie goes with it exactly when that asset is THIS user's clip of `clip_spec`. Without it a
|
||||
video YouTube gates signed out could never be clipped, while its download goes through."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return None
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
if cred is None:
|
||||
return None
|
||||
if asset.source_kind != "clip" or asset.source_ref != clipmod.asset_ref(user_id, clip_spec):
|
||||
return None
|
||||
return _readable(cred, user_id, host_key)
|
||||
|
||||
|
||||
def _readable(cred: DownloadCredential, user_id: int, host_key: str) -> str | None:
|
||||
raw = decrypt_optional(cred.cookies_enc)
|
||||
if not raw:
|
||||
log.warning(
|
||||
@@ -284,6 +307,14 @@ def resolve_cookiefile(
|
||||
return _write_cookiefile(raw, staging) if raw else None
|
||||
|
||||
|
||||
def resolve_clip_cookiefile(
|
||||
db: Session, user_id: int, url: str | None, asset: MediaAsset, clip_spec: dict, staging: Path
|
||||
) -> Path | None:
|
||||
"""`resolve_cookiefile` for an ad-hoc clip (`usable_clip_cookie`)."""
|
||||
raw = usable_clip_cookie(db, user_id, url, asset, clip_spec)
|
||||
return _write_cookiefile(raw, staging) if raw else None
|
||||
|
||||
|
||||
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
||||
"""Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path.
|
||||
|
||||
|
||||
+77
-119
@@ -188,6 +188,51 @@ def clip_duration(spec: dict, source_duration: int | None) -> int | None:
|
||||
|
||||
# --- ffmpeg: trim / crop --------------------------------------------------------------------
|
||||
|
||||
# Every libx264 / aac re-encode in this package uses these, so a re-encoded file is always the
|
||||
# phone-safe shape (`playable`): `-pix_fmt yuv420p` pins 8-bit 4:2:0 — without it libx264 keeps a
|
||||
# 10-bit or 4:4:4 source's pixel format and writes H.264 High 10 / High 4:4:4, which browsers decode
|
||||
# only patchily (the one safe H.264 is 8-bit 4:2:0).
|
||||
X264_ARGS = ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p"]
|
||||
AAC_ARGS = ["-c:a", "aac", "-b:a", "192k"]
|
||||
# The last step of every video filter chain a libx264 re-encode runs: 4:2:0 halves the chroma, so
|
||||
# libx264 refuses an odd width or height ("width not divisible by 2") — measured on an odd source
|
||||
# that is not 4:2:0 already (a 4:4:4 or RGB screen capture, 1365x767). A crop of a 4:2:0 source is
|
||||
# rounded to its chroma grid by ffmpeg itself; this holds every other path to the encoder to the same
|
||||
# even frame. Rounding down to even costs at most one pixel row/column.
|
||||
EVEN_FRAME = "scale=trunc(iw/2)*2:trunc(ih/2)*2"
|
||||
|
||||
# The demuxers ffmpeg/ffprobe may use to open a library file — single-file containers only. Without
|
||||
# it ffmpeg picks the demuxer from the file's CONTENT and NAME, and an upload's name is the user's:
|
||||
# a playlist-type demuxer (HLS, DASH, concat) opens whatever local paths its entries name, so a
|
||||
# crafted upload could copy another user's clip or a Plex file into its own converted asset. A file
|
||||
# no demuxer here can open fails the probe, and an unprobeable upload is a download-only 'file'.
|
||||
MEDIA_DEMUXERS = (
|
||||
"mov", # mp4, m4a, mov, 3gp
|
||||
"matroska", # mkv, webm
|
||||
"mpegts",
|
||||
"mpeg",
|
||||
"avi",
|
||||
"asf",
|
||||
"flv",
|
||||
"ogg",
|
||||
"mp3",
|
||||
"aac",
|
||||
"ac3",
|
||||
"eac3",
|
||||
"flac",
|
||||
"wav",
|
||||
"w64",
|
||||
"aiff",
|
||||
"caf",
|
||||
"amr",
|
||||
)
|
||||
INPUT_GUARD = ["-format_whitelist", ",".join(MEDIA_DEMUXERS)]
|
||||
# A downloaded thumbnail is the remote site's bytes under the suffix yt-dlp gave it (`.png`, `.webp`,
|
||||
# `.jpg` — `pipeline._THUMB_SUFFIXES`): only the single-image demuxers those open, so a "thumbnail"
|
||||
# that is really a playlist cannot pull a local file into the poster. Anything else fails, and the
|
||||
# poster falls back to a cut frame (`ensure_poster`).
|
||||
THUMB_GUARD = ["-format_whitelist", "png_pipe,jpeg_pipe,webp_pipe"]
|
||||
|
||||
|
||||
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
||||
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
|
||||
@@ -214,7 +259,7 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
||||
]
|
||||
if start:
|
||||
cmd += ["-ss", f"{start:.3f}"]
|
||||
cmd += ["-i", str(src)]
|
||||
cmd += [*INPUT_GUARD, "-i", str(src)]
|
||||
if end is not None:
|
||||
dur = end - start
|
||||
if dur > 0:
|
||||
@@ -222,21 +267,9 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
||||
|
||||
if reencode:
|
||||
# Input -ss + re-encode is frame-accurate (decodes from the prior keyframe, discards up to
|
||||
# the exact cut). A crop adds the filter; an accurate trim re-encodes with no filter.
|
||||
if crop:
|
||||
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
|
||||
cmd += [
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"20",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
]
|
||||
# the exact cut). A crop adds the filter; an accurate trim only evens the frame.
|
||||
cropf = f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}," if crop else ""
|
||||
cmd += ["-vf", cropf + EVEN_FRAME, *X264_ARGS, *AAC_ARGS]
|
||||
else:
|
||||
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
||||
|
||||
@@ -284,11 +317,14 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
||||
parts, labels = [], []
|
||||
for i, s in enumerate(segs):
|
||||
st, en = s["start_s"], s["end_s"]
|
||||
parts.append(f"[0:v]trim=start={st}:end={en},setpts=PTS-STARTPTS{cropf}[v{i}]")
|
||||
parts.append(
|
||||
f"[0:v]trim=start={st}:end={en},setpts=PTS-STARTPTS{cropf},{EVEN_FRAME}[v{i}]"
|
||||
)
|
||||
parts.append(f"[0:a]atrim=start={st}:end={en},asetpts=PTS-STARTPTS[a{i}]")
|
||||
labels.append(f"[v{i}][a{i}]")
|
||||
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
||||
cmd = head + [
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-filter_complex",
|
||||
@@ -297,26 +333,23 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
||||
"[v]",
|
||||
"-map",
|
||||
"[a]",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"20",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
*X264_ARGS,
|
||||
*AAC_ARGS,
|
||||
]
|
||||
else:
|
||||
listp = staging / "concat.txt"
|
||||
lines = []
|
||||
lines = ["ffconcat version 1.0"]
|
||||
for s in segs:
|
||||
lines.append(f"file '{_concat_quote(src)}'")
|
||||
# ffmpeg hands the list's whitelist (which must name `concat` to read the list) down to
|
||||
# the file it names, so a library file whose bytes are a concat list would be read as one
|
||||
# and open the paths inside it. Each entry's own whitelist replaces the inherited one.
|
||||
lines.append(f"option format_whitelist {INPUT_GUARD[1]}")
|
||||
lines.append(f"inpoint {s['start_s']}")
|
||||
lines.append(f"outpoint {s['end_s']}")
|
||||
prep[listp] = "\n".join(lines) + "\n"
|
||||
cmd = head + ["-f", "concat", "-safe", "0", "-i", str(listp), "-c", "copy"]
|
||||
guard = ["-format_whitelist", "concat," + INPUT_GUARD[1]]
|
||||
cmd = head + ["-f", "concat", "-safe", "0", *guard, "-i", str(listp), "-c", "copy"]
|
||||
|
||||
if out_ext in ("mp4", "m4a", "mov"):
|
||||
cmd += ["-movflags", "+faststart"]
|
||||
@@ -400,7 +433,7 @@ def escalate_kill(proc: subprocess.Popen, terminate, kill, grace_s: float, label
|
||||
try:
|
||||
proc.wait(timeout=grace_s)
|
||||
except subprocess.TimeoutExpired:
|
||||
# noqa TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
||||
# TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
||||
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
||||
|
||||
@@ -456,7 +489,7 @@ def run_ffmpeg(
|
||||
try:
|
||||
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
|
||||
except subprocess.TimeoutExpired: # pipes closed but the process lingers
|
||||
raise StatedFailure("ffmpeg did not exit after closing its output")
|
||||
raise StatedFailure("ffmpeg did not exit after closing its output") from None
|
||||
except BaseException:
|
||||
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
|
||||
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
|
||||
@@ -474,14 +507,7 @@ def run_ffmpeg(
|
||||
raise StatedFailure(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
|
||||
|
||||
# --- ffmpeg: universal browser playability --------------------------------------------------
|
||||
# A downloaded file must play in EVERY browser, whatever the source shipped. Universal = H.264
|
||||
# video + AAC/MP3 audio in an mp4 container: H.264 is the only video codec all browsers decode
|
||||
# (VP9/AV1 fail on Safari/iOS; HEVC fails on Chrome/Firefox), and mp4/AAC is the safe pairing.
|
||||
# Anything else is fixed post-download: re-encode the offending stream(s), or just remux when the
|
||||
# codecs are already fine but the container isn't.
|
||||
_BROWSER_SAFE_VCODEC = ("h264",)
|
||||
_BROWSER_SAFE_ACODEC = ("aac", "mp3")
|
||||
# --- ffprobe ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_ffprobe_json(
|
||||
@@ -493,10 +519,10 @@ def run_ffprobe_json(
|
||||
) -> dict | None:
|
||||
"""One ffprobe call → its parsed JSON dict, or None on any failure (timeout, non-zero exit,
|
||||
missing binary, unparseable output). The SINGLE ffprobe-JSON-invocation point in the downloads
|
||||
package: the browser-playability codec probe here and the shared media probe (`probe.probe_media`,
|
||||
used by both the download worker's finalize and the upload ingest) both go through it, so the
|
||||
package: the shared media probe (`probe.probe_json`: the stored facts and the phone-safe rule) goes
|
||||
through it, so the
|
||||
subprocess flags/timeout/error handling live in one place instead of drifting copies."""
|
||||
args = ["ffprobe", "-v", "error"]
|
||||
args = ["ffprobe", "-v", "error", *INPUT_GUARD]
|
||||
if show_streams:
|
||||
args.append("-show_streams")
|
||||
if show_format:
|
||||
@@ -517,35 +543,6 @@ def run_ffprobe_json(
|
||||
return None
|
||||
|
||||
|
||||
def _codecs_from_streams(streams: list) -> tuple[str | None, str | None]:
|
||||
"""(video_codec, audio_codec) lowercased from an ffprobe JSON stream list — the first of each kind.
|
||||
|
||||
Parsed from the JSON `streams` list, NOT ffprobe's flat `default=nw=1` output: there the fields
|
||||
arrive in ffprobe's own struct order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line
|
||||
"remember the last codec_type" parser attributes every name to the PREVIOUS stream. Measured on real
|
||||
downloads: an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
|
||||
believed the video stream was AAC and re-encoded a perfectly fine file on every download. Reading each
|
||||
stream dict by its own `codec_type` is immune to that ordering."""
|
||||
vcodec = acodec = None
|
||||
for s in streams:
|
||||
name = (s.get("codec_name") or "").lower() or None
|
||||
ctype = (s.get("codec_type") or "").lower()
|
||||
if ctype == "video" and vcodec is None:
|
||||
vcodec = name
|
||||
elif ctype == "audio" and acodec is None:
|
||||
acodec = name
|
||||
return vcodec, acodec
|
||||
|
||||
|
||||
def probe_codecs(path: Path) -> tuple[str | None, str | None]:
|
||||
"""Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing
|
||||
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
|
||||
data = run_ffprobe_json(path, entries="stream=codec_type,codec_name")
|
||||
if data is None:
|
||||
return None, None
|
||||
return _codecs_from_streams(data.get("streams") or [])
|
||||
|
||||
|
||||
def probe_duration(path: Path) -> int:
|
||||
"""Media duration in whole seconds via ffprobe, or 0 if unknown — a progress denominator for a
|
||||
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
|
||||
@@ -555,6 +552,7 @@ def probe_duration(path: Path) -> int:
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
*INPUT_GUARD,
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
@@ -577,49 +575,6 @@ def probe_duration(path: Path) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def browser_transcode_flags(
|
||||
vcodec: str | None, acodec: str | None, is_mp4: bool
|
||||
) -> tuple[bool, bool, bool]:
|
||||
"""Decide what (if anything) makes a file universally browser-playable. Returns
|
||||
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
|
||||
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
|
||||
rv = bool(vcodec) and vcodec not in _BROWSER_SAFE_VCODEC
|
||||
ra = bool(acodec) and acodec not in _BROWSER_SAFE_ACODEC
|
||||
return (rv or ra or not is_mp4), rv, ra
|
||||
|
||||
|
||||
def build_browser_safe_cmd(
|
||||
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool, threads: int | None = None
|
||||
) -> list[str]:
|
||||
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
||||
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
||||
pipe:1` streams machine-readable progress for run_ffmpeg. `threads` caps the encoder so a
|
||||
re-encode can't claim every core on the shared box (see the worker-child de-prioritization)."""
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-nostats",
|
||||
"-i",
|
||||
str(src),
|
||||
]
|
||||
cmd += (
|
||||
["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"]
|
||||
if reencode_video
|
||||
else ["-c:v", "copy"]
|
||||
)
|
||||
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
||||
if threads and threads > 0:
|
||||
cmd += ["-threads", str(threads)]
|
||||
cmd += ["-movflags", "+faststart", str(dest)]
|
||||
return cmd
|
||||
|
||||
|
||||
# --- filmstrip (editor scrub track) ---------------------------------------------------------
|
||||
|
||||
_SB_COLS = 12
|
||||
@@ -655,6 +610,7 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
@@ -678,6 +634,7 @@ def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
||||
"-nostdin",
|
||||
"-ss",
|
||||
f"{max(0.0, at_s):.3f}",
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
@@ -699,6 +656,7 @@ def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
*THUMB_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
@@ -734,7 +692,7 @@ def _step_failure(err: BaseException) -> str:
|
||||
|
||||
|
||||
def _is_jpeg(path: Path) -> bool:
|
||||
"""True if the file starts with the JPEG SOI marker (FF D8 FF)."""
|
||||
"""True if the file starts with the JPEG SOI marker (0xFF 0xD8 0xFF)."""
|
||||
try:
|
||||
with path.open("rb") as f:
|
||||
return f.read(3) == b"\xff\xd8\xff"
|
||||
@@ -774,7 +732,7 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
||||
return dest if dest.exists() else None
|
||||
|
||||
|
||||
def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
def ensure_poster(asset, files_root: str, timeout_s: int = 30) -> str | None:
|
||||
"""Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL
|
||||
like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's
|
||||
`<base>.jpg` sidecar — so Plex uses it too and `delete_asset_files` cleans it up — and returned
|
||||
@@ -782,7 +740,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
short clip) dodges black intro frames."""
|
||||
if not asset.rel_path:
|
||||
return None
|
||||
root = Path(download_root)
|
||||
root = Path(files_root)
|
||||
src = root / asset.rel_path
|
||||
if not src.exists():
|
||||
return None
|
||||
@@ -798,7 +756,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
return rel if dest.exists() else None
|
||||
|
||||
|
||||
def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | None:
|
||||
def ensure_storyboard(asset, files_root: str, timeout_s: int = 90) -> dict | None:
|
||||
"""Return the filmstrip meta for a ready source asset, generating the sprite on first use.
|
||||
|
||||
Best-effort + time-bounded: for a very long/high-res source the one-pass decode can be slow,
|
||||
@@ -810,7 +768,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict |
|
||||
if not asset.rel_path:
|
||||
return None
|
||||
geom = storyboard_geometry(asset.duration_s)
|
||||
root = Path(download_root)
|
||||
root = Path(files_root)
|
||||
rel = f".storyboards/asset-{asset.id}.jpg"
|
||||
dest = root / rel
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ explanation (and decides whether a retry makes sense). The raw text stays availa
|
||||
"details" disclosure. This is the same string-matching approach the worker already trusts for
|
||||
`_RETRIABLE_MARKERS` — kept here as the single source of truth for "what kind of failure is this".
|
||||
|
||||
**The failure catalog** (`_CATALOG`, E11.5). A category is generic: "Sign-in required" cannot tell a
|
||||
**The failure catalog** (`_CATALOG`). A category is generic: "Sign-in required" cannot tell a
|
||||
Facebook user that the cookie they already gave was refused, and yt-dlp words several different
|
||||
Facebook walls identically (`Cannot parse data`). A catalog entry is scoped to ONE host and may also
|
||||
depend on whether a cookie was attached and on the kind of link, and names a `detail` code the UI has
|
||||
@@ -37,9 +37,10 @@ from app.downloads.credentials import host_key_for_url
|
||||
|
||||
# Worker-authored strings that this module owns because each is coupled to `_RULES`.
|
||||
#
|
||||
# QUOTA_FULL_MESSAGE is raised by the worker and must classify back to `quota_exceeded`; its rule
|
||||
# below matches a fragment of it, and test_download_errors pins the pair, because a message and its
|
||||
# rule drifting apart is exactly how a category silently becomes unreachable.
|
||||
# QUOTA_FULL_MESSAGE and the three *_TOO_BIG_MESSAGEs are raised by the worker and must classify
|
||||
# back to `quota_exceeded`; their rules below match a fragment of each, and test_download_errors pins
|
||||
# each one, because a message and its rule drifting apart is exactly how a
|
||||
# category silently becomes unreachable.
|
||||
#
|
||||
# NETWORK_REDACTED_MESSAGE REPLACES a message after classification (`user_error_text`). It is written
|
||||
# to the shared asset too, and a second job that inherits that failed asset re-diagnoses the stored
|
||||
@@ -49,10 +50,24 @@ from app.downloads.credentials import host_key_for_url
|
||||
# "produced no output file" have markers here; the urlguard refusal deliberately has none and
|
||||
# classifies as `unknown` — it is a refusal, not a diagnosis, and no category would say more.
|
||||
#
|
||||
# A source adapter's verdicts on a link (`SourceAdapter.resolve_url`, E11.5 S4) are adapters.py's
|
||||
# A source adapter's verdicts on a link (`SourceAdapter.resolve_url`) are adapters.py's
|
||||
# constants, imported above; each has a rule below that gives its category back, and a catalog entry
|
||||
# that gives the site's sentence.
|
||||
QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
|
||||
# A download, a cut clip, or an upload's converted file, held to the space left, which is not zero:
|
||||
# "full" would be untrue. All three classify by the one phrase they share.
|
||||
DOWNLOAD_TOO_BIG_MESSAGE = (
|
||||
"The video is larger than the download storage you have left — free up space, then try again."
|
||||
)
|
||||
CLIP_TOO_BIG_MESSAGE = (
|
||||
"The clip is larger than the download storage you have left — "
|
||||
"free up space or cut a shorter clip, then try again."
|
||||
)
|
||||
CONVERTED_TOO_BIG_MESSAGE = (
|
||||
"The converted file is larger than the download storage you have left — "
|
||||
"free up space, then try again."
|
||||
)
|
||||
UPLOAD_MISSING_MESSAGE = "The uploaded file is missing."
|
||||
NETWORK_REDACTED_MESSAGE = "The server could not reach that address."
|
||||
|
||||
# (marker, category) — first match wins, so order most-specific → most-general. Matching is
|
||||
@@ -61,6 +76,7 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
# Our own refusal: the worker found the user already at their storage cap (SEC-13). Placed
|
||||
# first — it is our wording, so it can never be a false match on an upstream message.
|
||||
("download storage is full", "quota_exceeded"),
|
||||
("larger than the download storage you have left", "quota_exceeded"),
|
||||
# Our own redaction of a transport failure (see NETWORK_REDACTED_MESSAGE) — our wording too.
|
||||
(NETWORK_REDACTED_MESSAGE.lower(), "network"),
|
||||
# Our own verdicts on a post link (adapters.py) — our wording too: its page holds no single video,
|
||||
@@ -98,6 +114,8 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
# An edit's source download went away — our own wording, and MORE specific than the generic
|
||||
# "no longer available" below, so it must be matched first.
|
||||
("source download is no longer available", "source_missing"),
|
||||
# An upload's original went away before its conversion ran — our wording (UPLOAD_MISSING_MESSAGE).
|
||||
(UPLOAD_MISSING_MESSAGE.lower().rstrip("."), "upload_missing"),
|
||||
("source file is missing", "source_missing"),
|
||||
# Private / deleted / removed / terminated.
|
||||
("private video", "unavailable"),
|
||||
@@ -126,6 +144,13 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
("postprocessing", "postprocess"),
|
||||
("produced no output file", "postprocess"),
|
||||
("no output file", "postprocess"),
|
||||
# yt-dlp's own verdict that its extractor did not understand the site's answer: it appends this
|
||||
# request for a bug report to an UNEXPECTED extraction failure, never to an expected one (a private,
|
||||
# removed or geo-blocked video says so in words matched above). Measured 2026-10-12: a TikTok link
|
||||
# failed with it on every attempt under yt-dlp 2026.07.04 and downloads under 2026.08.19 — the
|
||||
# site changed and a newer yt-dlp is the cure. After the media-format and post-processing rules,
|
||||
# whose texts can carry the same request; before the network ones, which are about the transport.
|
||||
("please report this issue", "site_changed"),
|
||||
# Network / transport.
|
||||
("unable to download", "network"),
|
||||
("http error", "network"),
|
||||
@@ -263,6 +288,15 @@ def classify(msg: str | None) -> str:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def classify_conversion(msg: str | None) -> str:
|
||||
"""The category of an upload's failed conversion. The generic ffmpeg categories speak of a
|
||||
download ("the video downloaded but…"), which an upload never was: a failed encode or an
|
||||
unrecognised failure of one is `convert_failed`; a category of its own (a full quota, a missing
|
||||
original) stays as it is."""
|
||||
code = classify(msg)
|
||||
return "convert_failed" if code in ("postprocess", "unknown") else code
|
||||
|
||||
|
||||
def diagnose(
|
||||
msg: str | None, *, url: str | None = None, cookie_attached: bool = False
|
||||
) -> Diagnosis:
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
"""The folders of the Files library — where each of a user's files sits.
|
||||
|
||||
A folder is a row, never a directory: on disk a file stays where the download layout put it (one
|
||||
shared asset can be held by several users), so a rename or a move is a column update. Three kinds:
|
||||
|
||||
- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips,
|
||||
uploads, one per other site). Its `source_key` names the origin; the client renders its name
|
||||
from the key, in the reader's language.
|
||||
- an OWN folder is the user's, with the name they gave it.
|
||||
- the HUB folder (one per user, made when they first give a machine a token for it, `hub_folder`) is
|
||||
where they swap files with that machine. Its name, like a source folder's, is the client's; a file
|
||||
may be moved in like into an own folder, and it keeps a file until it is deleted.
|
||||
|
||||
Every file of a user is in exactly one folder. The folder a new file lands in is decided here
|
||||
(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file
|
||||
moves only into one of the user's own folders, their Hub folder, or back to its own source folder
|
||||
(`move`): a YouTube file in the TikTok folder would be a lie the folder's name tells.
|
||||
|
||||
Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
|
||||
folder was made; the user can change it), Uploads, own folders and the Hub keep it until it is deleted. A
|
||||
done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user
|
||||
kept it — and its `ready_at`, the moment it finished (`stamp_done`).
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import StrEnum
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, FileFolder, MediaAsset
|
||||
|
||||
|
||||
class FolderKind(StrEnum):
|
||||
SOURCE = "source"
|
||||
OWN = "own"
|
||||
HUB = "hub"
|
||||
|
||||
|
||||
class SourceKey(StrEnum):
|
||||
"""The fixed origins. Any other site is `site:<host>` (see `site_key`)."""
|
||||
|
||||
YOUTUBE = "youtube"
|
||||
PLEX = "plex"
|
||||
UPLOAD = "upload"
|
||||
|
||||
|
||||
SITE_PREFIX = "site:"
|
||||
|
||||
# A host's leading label that only says "the same site, another front door".
|
||||
_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
|
||||
_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
|
||||
|
||||
|
||||
def site_key(url: str) -> str:
|
||||
"""The source key of a page on some site: `site:<host>` without its front door (www., m.),
|
||||
or the YouTube key for a YouTube address that arrived as a plain URL."""
|
||||
host = (urlsplit(url).hostname or "").lower().rstrip(".")
|
||||
for door in _FRONT_DOORS:
|
||||
if host.startswith(door) and host.count(".") > 1:
|
||||
host = host[len(door) :]
|
||||
break
|
||||
if host in _YOUTUBE_HOSTS:
|
||||
return SourceKey.YOUTUBE
|
||||
return f"{SITE_PREFIX}{host or 'unknown'}"
|
||||
|
||||
|
||||
def source_key_for(db: Session, job: DownloadJob) -> str:
|
||||
"""The origin a job's file came from — the source folder it belongs in.
|
||||
|
||||
The one recorded when it was placed, else: youtube → YouTube; a clip → the clip's source
|
||||
(YouTube or Plex); an upload → Uploads; a URL → its site; an edit → wherever its parent's file
|
||||
came from (its parent job, or, once that is deleted, the parent's asset)."""
|
||||
if job.source_key:
|
||||
return job.source_key
|
||||
kind = job.source_kind
|
||||
if kind == "youtube":
|
||||
return SourceKey.YOUTUBE
|
||||
if kind == "upload":
|
||||
return SourceKey.UPLOAD
|
||||
if kind == "clip":
|
||||
return (
|
||||
SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE
|
||||
)
|
||||
if kind == "url":
|
||||
return site_key(job.source_ref)
|
||||
if kind == "edit":
|
||||
parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None
|
||||
if parent is not None and parent.id != job.id:
|
||||
return source_key_for(db, parent)
|
||||
asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None
|
||||
if asset is not None:
|
||||
if asset.source_kind == "youtube":
|
||||
return SourceKey.YOUTUBE
|
||||
if asset.source_kind == "upload":
|
||||
return SourceKey.UPLOAD
|
||||
if asset.source_kind == "url":
|
||||
return site_key(asset.source_ref)
|
||||
return SourceKey.UPLOAD
|
||||
|
||||
|
||||
def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
|
||||
"""The user's source folder for `key`, made on first use. A race on the unique (user, key) is
|
||||
settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
|
||||
stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
|
||||
folder = db.execute(stmt).scalar_one_or_none()
|
||||
if folder is not None:
|
||||
return folder
|
||||
retention = (
|
||||
None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days")
|
||||
)
|
||||
folder = FileFolder(
|
||||
user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention
|
||||
)
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(folder)
|
||||
db.flush()
|
||||
# silent-ok: the expected outcome of the race the savepoint exists for — another request made
|
||||
# the same folder first; re-selecting it is the whole handling.
|
||||
except IntegrityError:
|
||||
folder = db.execute(stmt).scalar_one()
|
||||
return folder
|
||||
|
||||
|
||||
def hub_folder(db: Session, user_id: int) -> FileFolder:
|
||||
"""The user's Hub folder, made on first use; a race on its one-per-user index is settled like
|
||||
`source_folder`'s."""
|
||||
stmt = select(FileFolder).where(
|
||||
FileFolder.user_id == user_id, FileFolder.kind == FolderKind.HUB
|
||||
)
|
||||
folder = db.execute(stmt).scalar_one_or_none()
|
||||
if folder is not None:
|
||||
return folder
|
||||
folder = FileFolder(user_id=user_id, kind=FolderKind.HUB)
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(folder)
|
||||
db.flush()
|
||||
# silent-ok: another request made the Hub folder first; re-selecting it is the whole handling.
|
||||
except IntegrityError:
|
||||
folder = db.execute(stmt).scalar_one()
|
||||
return folder
|
||||
|
||||
|
||||
def place(db: Session, job: DownloadJob) -> None:
|
||||
"""Put a new job's file in its source folder, and record that origin on it. Called wherever a
|
||||
job is created; the job must carry its `user_id`, `source_kind` and source refs already."""
|
||||
key = source_key_for(db, job)
|
||||
job.source_key = key
|
||||
job.folder_id = source_folder(db, job.user_id, key).id
|
||||
|
||||
|
||||
def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
|
||||
"""When a file that finishes now goes: its folder's retention from now, or never (kept, or a
|
||||
folder that keeps files)."""
|
||||
if kept or folder_id is None:
|
||||
return None
|
||||
# Read under a share lock, held to the caller's commit: a retention change (`set_retention` locks
|
||||
# the folder) waits for this file and restamps it too, or this read waits for the change and
|
||||
# takes the new value — never the old value stamped on a file the change did not see.
|
||||
days = db.execute(
|
||||
select(FileFolder.retention_days)
|
||||
.where(FileFolder.id == folder_id)
|
||||
.with_for_update(read=True)
|
||||
).scalar()
|
||||
return None if days is None else datetime.now(timezone.utc) + timedelta(days=days)
|
||||
|
||||
|
||||
def stamp_done(db: Session, job: DownloadJob) -> None:
|
||||
"""Stamp a job that just became done (in this session): when it did, and its expiry."""
|
||||
job.ready_at = datetime.now(timezone.utc)
|
||||
job.expires_at = done_expiry(db, job.folder_id, job.kept)
|
||||
job.expiry_warned = False
|
||||
|
||||
|
||||
class FolderError(Exception):
|
||||
"""A folder change the rules refuse; `reason` is the refusal the route answers with."""
|
||||
|
||||
def __init__(self, reason: str) -> None:
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
|
||||
"""The user's folder `folder_id` that a file is about to be put in — held (FOR SHARE) until the
|
||||
caller's transaction ends, so a delete of the folder waits for the file and then takes it home
|
||||
with the rest (`delete_own` locks it FOR UPDATE), never failing on its foreign key. `hold=False`
|
||||
only looks (a check ahead of a long wait, which must not hold a lock through it). Another user's
|
||||
folder answers like a missing one."""
|
||||
stmt = select(FileFolder).where(FileFolder.id == folder_id)
|
||||
if hold:
|
||||
stmt = stmt.with_for_update(read=True)
|
||||
folder = db.execute(stmt.execution_options(populate_existing=True)).scalar_one_or_none()
|
||||
if folder is None or folder.user_id != user_id:
|
||||
raise FolderError("file_folder_gone")
|
||||
return folder
|
||||
|
||||
|
||||
def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
|
||||
"""The folder an upload asked to land in, if it may: one of the user's own folders, their Hub
|
||||
folder, or their Uploads folder. A file put in a site's folder would claim an origin it does not
|
||||
have."""
|
||||
folder = own_folder(db, user_id, folder_id, hold=hold)
|
||||
if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD:
|
||||
raise FolderError("file_folder_not_its_source")
|
||||
return folder
|
||||
|
||||
|
||||
def restamp(db: Session, job: DownloadJob) -> None:
|
||||
"""Start a file's clock again from now: it moved to another folder, or its user stopped keeping
|
||||
it. A file not done yet gets its expiry when it finishes (`stamp_done`), so it has none now."""
|
||||
done = job.status == JobStatus.DONE
|
||||
job.expires_at = done_expiry(db, job.folder_id, job.kept) if done else None
|
||||
job.expiry_warned = False
|
||||
|
||||
|
||||
def set_kept(db: Session, job: DownloadJob, kept: bool) -> None:
|
||||
"""Keep a file until its user deletes it, or hand it back to its folder's retention."""
|
||||
job.kept = kept
|
||||
restamp(db, job)
|
||||
|
||||
|
||||
def move(db: Session, job: DownloadJob, folder: FileFolder) -> None:
|
||||
"""Put a file in `folder` — one of its user's own folders, their Hub folder, or the source folder
|
||||
it came from.
|
||||
The caller has checked that `folder` is the job's user's."""
|
||||
if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job):
|
||||
raise FolderError("file_folder_not_its_source")
|
||||
if folder.id == job.folder_id:
|
||||
return
|
||||
job.folder_id = folder.id
|
||||
restamp(db, job)
|
||||
|
||||
|
||||
def _hold_for_change(db: Session, folder: FileFolder) -> None:
|
||||
"""Lock `folder` FOR UPDATE and read it afresh into the same object: a change waits out a file
|
||||
being put in it or stamped from it right now (`own_folder`, `done_expiry`), and works from what
|
||||
another change that landed first left, not from what the caller read before the wait. The row is
|
||||
consumed on purpose: an ORM result refreshes the object only as its rows are fetched."""
|
||||
fresh = db.execute(
|
||||
select(FileFolder)
|
||||
.where(FileFolder.id == folder.id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if fresh is None:
|
||||
raise FolderError("file_folder_gone")
|
||||
|
||||
|
||||
def set_retention(db: Session, folder: FileFolder, days: int | None) -> int | None:
|
||||
"""Change how long `folder` keeps a file (None = until deleted). The done files already in it
|
||||
move by the difference — each keeps the age it has — but none ends up closer than the GC's
|
||||
warning window, so a shortened retention still warns before it takes anything. Returns the
|
||||
retention it replaced (equal to `days` when nothing changed)."""
|
||||
_hold_for_change(db, folder) # so the file arriving now is among the ones restamped below
|
||||
old = folder.retention_days
|
||||
if old == days:
|
||||
return old
|
||||
folder.retention_days = days
|
||||
now = datetime.now(timezone.utc)
|
||||
floor = now + timedelta(days=sysconfig.get_int(db, "download_gc_grace_days"))
|
||||
# Locked as they are read: one moved out or deleted meanwhile is no longer this folder's to time.
|
||||
jobs = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(
|
||||
DownloadJob.folder_id == folder.id,
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalars()
|
||||
for job in jobs:
|
||||
if days is None:
|
||||
job.expires_at = None
|
||||
elif old is None or job.expires_at is None:
|
||||
job.expires_at = max(now + timedelta(days=days), floor)
|
||||
else:
|
||||
job.expires_at = max(job.expires_at + timedelta(days=days - old), floor)
|
||||
job.expiry_warned = False
|
||||
return old
|
||||
|
||||
|
||||
def delete_own(db: Session, folder: FileFolder) -> int:
|
||||
"""Delete one of a user's own folders. Its files are not deleted: each goes back to the source
|
||||
folder it came from, its clock starting there now. Returns how many moved."""
|
||||
if folder.kind != FolderKind.OWN:
|
||||
raise FolderError("file_folder_not_own")
|
||||
_hold_for_change(db, folder) # so the file arriving now is taken home too
|
||||
# Locked as they are read: a file moved out meanwhile stays where it went, and one deleted
|
||||
# meanwhile is not written back (its gone row would fail the delete).
|
||||
jobs = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.folder_id == folder.id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for job in jobs:
|
||||
job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
|
||||
restamp(db, job)
|
||||
db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete
|
||||
db.delete(folder)
|
||||
return len(jobs)
|
||||
@@ -13,7 +13,7 @@ spec shape (all keys optional; defaults + browser-playability clamp applied by `
|
||||
vcodec always coerced to null → H.264 selected (see `normalize`); no opt-out
|
||||
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
|
||||
|
||||
`normalize` clamps container/vcodec/audio_format so EVERY download plays in-browser (E7); the
|
||||
`normalize` clamps container/vcodec/audio_format so EVERY download plays in-browser; the
|
||||
`container`/`vcodec` keys survive only so an old stored profile deserializes without error.
|
||||
"""
|
||||
|
||||
@@ -105,6 +105,28 @@ def log_safe(text: object) -> str:
|
||||
return _PATH_RE.sub("…", text)
|
||||
|
||||
|
||||
# yt-dlp's HTTP downloader, refusing a file announced larger than `max_filesize`, logs this line and
|
||||
# RETURNS — no exception and no file (measured; test_formats holds it against the pinned yt-dlp).
|
||||
_OVER_CAP_LINE = "File is larger than max-filesize"
|
||||
_OVER_CAP: contextvars.ContextVar[list[bool] | None] = contextvars.ContextVar(
|
||||
"formats_over_cap", default=None
|
||||
)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def watching_size_cap() -> Iterator[list[bool]]:
|
||||
"""While yt-dlp works inside it: a one-slot flag, set when it refused a file over the size cap
|
||||
(`build_ydl_opts(max_bytes=…)`). It then returns as if it had downloaded, and all the caller
|
||||
would find is no file — a processing fault, to anyone reading it, for what is the owner's
|
||||
storage. A context variable, like `hiding`: yt-dlp's logger calls run on the extraction's thread."""
|
||||
flag = [False]
|
||||
token = _OVER_CAP.set(flag)
|
||||
try:
|
||||
yield flag
|
||||
finally:
|
||||
_OVER_CAP.reset(token)
|
||||
|
||||
|
||||
class _YdlLog:
|
||||
"""yt-dlp's logger protocol, pointed at ours.
|
||||
|
||||
@@ -124,6 +146,8 @@ class _YdlLog:
|
||||
is the one line whose level should say so."""
|
||||
|
||||
def _emit(self, level: int, msg: str) -> None:
|
||||
if _OVER_CAP_LINE in msg and (over := _OVER_CAP.get()) is not None:
|
||||
over[0] = True
|
||||
# The line goes as an ARGUMENT, not as the format string. yt-dlp writes progress as
|
||||
# `[download] 53.2% of 12.00MiB`, and while stdlib logging only applies `%` formatting to
|
||||
# a record that has args, this way the question never arises — and it is the shape ruff's
|
||||
@@ -182,7 +206,9 @@ _DEFAULTS = {
|
||||
# v4: EVERY download is clamped to the mp4/H.264/AAC compat profile (no more mkv/webm/VP9/AV1/Opus
|
||||
# opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped
|
||||
# the v3 net — must re-derive through the current browser-safe guarantee.
|
||||
_SIG_VERSION = 4
|
||||
# v5: the phone-safe rule (`playable`) also judges the pixel format, so a cached 10-bit / 4:4:4 H.264
|
||||
# asset (which v4 let through as "H.264") re-derives as 8-bit 4:2:0 instead of being reused.
|
||||
_SIG_VERSION = 5
|
||||
# Audio-only downloads version on their own, so a change to the audio bytes doesn't re-download every
|
||||
# cached video. v5: an extracted .mp3/.m4a is stored as itself instead of remuxed into an .mp4 that
|
||||
# iPad's WebKit refuses to play, so a cached remuxed audio asset must re-derive.
|
||||
@@ -217,14 +243,15 @@ def normalize(spec: dict) -> dict:
|
||||
# Audio-only files can't embed subtitles/thumbnails as video; keep the flags meaningful.
|
||||
if s["mode"] == "a":
|
||||
s["embed_subs"] = False
|
||||
# Browser-playability guarantee (E7): the Download module exists to download → share → PLAY
|
||||
# Browser-playability guarantee: the Download module exists to download → share → PLAY
|
||||
# in-browser, so every output must play in EVERY browser, iOS/Safari/WebKit included. The only
|
||||
# universally-safe combo is mp4 + H.264 + AAC/MP3, so we CLAMP the spec to it HERE — the single
|
||||
# point every path (picker, stored profile, raw API) flows through — instead of only hiding
|
||||
# options in the UI. Non-mp4 containers (mkv doesn't play at all; webm/VP9/AV1 fail iOS) and any
|
||||
# explicit vcodec are coerced to the compat profile; Opus audio (Safari-flaky as a bare file)
|
||||
# falls back to AAC/m4a. A source that exposes no H.264 is still caught by the worker's
|
||||
# browser-safe re-encode (`_ensure_browser_playable`), so playback is guaranteed end-to-end.
|
||||
# universally-safe combo is mp4 + H.264 + AAC (an .mp3/.m4a for audio), so we CLAMP the spec to
|
||||
# it HERE — the single point every path (picker, stored profile, raw API) flows through — instead
|
||||
# of only hiding options in the UI. Non-mp4 containers (mkv plays nowhere on iOS; AV1 fails there,
|
||||
# and webm/VP9 plays only on recent devices) and any explicit vcodec are coerced to the compat
|
||||
# profile; Opus audio falls back to AAC/m4a. A source that exposes no H.264 (or a 10-bit one) is
|
||||
# still caught by the worker's phone-safe step (`downloads.playable`), so playback is guaranteed
|
||||
# end-to-end.
|
||||
s["container"] = "mp4"
|
||||
s["vcodec"] = None
|
||||
if s["audio_format"] not in ("m4a", "mp3"):
|
||||
@@ -274,12 +301,15 @@ def build_ydl_opts(
|
||||
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
||||
generic options below. Everything here is source-agnostic.
|
||||
|
||||
`cookiefile` is the acting user's per-host cookies.txt path (S6), attached ONLY by the worker
|
||||
`cookiefile` is the acting user's per-host cookies.txt path, attached ONLY by the worker
|
||||
after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra`
|
||||
merge so a source adapter can never clobber it.
|
||||
|
||||
`max_bytes` (SEC-13) is the caller's remaining storage budget; yt-dlp refuses a format larger
|
||||
than it rather than filling the disk and only then being caught by the quota accounting."""
|
||||
`max_bytes` (SEC-13) is the caller's remaining storage budget; yt-dlp's HTTP downloader refuses
|
||||
a file announced larger than it rather than filling the disk and only then being caught by the
|
||||
quota accounting. Its ffmpeg downloader (a `download_ranges` fetch) does not read it, nor does
|
||||
a fragmented download (YouTube's formats: measured, 718 MB fetched with 1 MB left) — so the
|
||||
pipeline also holds every produced file to what is left by its real size."""
|
||||
s = normalize(spec)
|
||||
h = s["max_height"]
|
||||
opts: dict = {
|
||||
@@ -329,7 +359,7 @@ def build_ydl_opts(
|
||||
if max_bytes and max_bytes > 0:
|
||||
opts["max_filesize"] = max_bytes
|
||||
|
||||
# The acting user's per-host cookie (S6). AFTER the adapter merge on purpose: an adapter must not
|
||||
# The acting user's per-host cookie. AFTER the adapter merge on purpose: an adapter must not
|
||||
# be able to override the authenticated session the worker resolved for this specific user.
|
||||
if cookiefile:
|
||||
opts["cookiefile"] = cookiefile
|
||||
@@ -351,7 +381,7 @@ def build_ydl_opts(
|
||||
# inside mp4 — a VP9/AV1-in-mp4 file shows a broken player there — so H.264 is selected at the
|
||||
# SELECTION level, not merely preferred: a sort key can't conjure an H.264 stream a source
|
||||
# doesn't expose as video-only. A source that ships only non-H.264 is caught after download by
|
||||
# the worker's browser-safe re-encode (`_ensure_browser_playable`).
|
||||
# the worker's phone-safe step (`downloads.playable`).
|
||||
if s["mode"] == "v":
|
||||
# Video-only: keep it audio-free. Prefer an H.264 video-only stream, then any video-only,
|
||||
# and only fall to a progressive (audio-bearing) `best` as a last resort.
|
||||
|
||||
+232
-96
@@ -1,82 +1,167 @@
|
||||
"""Download retention garbage collector (a scheduler job).
|
||||
|
||||
Five passes each run:
|
||||
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
|
||||
once (gc_notified flag) so a download isn't silently deleted.
|
||||
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
||||
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
||||
ready assets until back under it.
|
||||
Six passes each run:
|
||||
1. Pre-expiry warning — a done file whose `expires_at` (its folder's retention, `folders`) falls
|
||||
within the grace window, and that its user did not keep, warns its user once (`expiry_warned`).
|
||||
2. Expiry — such a file past its `expires_at` leaves the library: the job is deleted and its hold
|
||||
released, so the shared file goes with its last holder.
|
||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed ready
|
||||
assets (never an upload, never one a user kept) until back under it; each holder's file leaves
|
||||
their library, with a notice.
|
||||
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
|
||||
passes 1-3 skip them; benign row-count hygiene).
|
||||
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
|
||||
re-download renames an asset, e.g. old accented channel dirs), skipping system trees and
|
||||
anything newer than an hour so an in-flight download is never touched.
|
||||
re-download renames an asset, e.g. old accented channel dirs, or an upload's stranded file).
|
||||
It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
|
||||
so an in-flight download is never touched.
|
||||
6. Token expiry warning — a machine's token for the library about to run out warns its user once
|
||||
(`apitokens.warn_expiring`): the library's other clock.
|
||||
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
Runs in the API process (which mounts FILES_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app import apitokens, logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
||||
# The one dot-prefixed top-level tree the orphan sweep enters. The others are not part of the asset
|
||||
# layout (user clips, Plex HLS/img caches, in-progress staging, editor storyboards) and are skipped.
|
||||
_UPLOAD_TREE = ".uploads"
|
||||
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
|
||||
|
||||
|
||||
def _holders(db: Session, asset_id: int) -> list[int]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
|
||||
).scalars()
|
||||
def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str) -> None:
|
||||
title = job.display_name or (asset.title if asset else None) or "download"
|
||||
create_notification(
|
||||
db,
|
||||
user_id=job.user_id,
|
||||
type=ntype,
|
||||
title=title, # English fallback; the client renders a translated message from data
|
||||
data={
|
||||
"kind": ntype,
|
||||
"title": title,
|
||||
"video_id": asset.source_ref if asset else None,
|
||||
"job_id": job.id,
|
||||
},
|
||||
commit=False,
|
||||
)
|
||||
|
||||
|
||||
def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
title = asset.title or "download"
|
||||
for uid in _holders(db, asset.id):
|
||||
create_notification(
|
||||
db,
|
||||
user_id=uid,
|
||||
type=ntype,
|
||||
title=title, # English fallback; the client renders a translated message from data
|
||||
data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
|
||||
commit=False,
|
||||
def _relock_expired(
|
||||
db: Session, job_id: int, asset_id: int | None, now: datetime
|
||||
) -> DownloadJob | None:
|
||||
"""Re-read an expired job under the lock order the user routes take (`own_job_for_update`: the
|
||||
asset, then the job) and say whether it is still to go. Between the pass's SELECT and here a
|
||||
user may have deleted or re-downloaded it (dropping it again would release its asset a second
|
||||
time, freeing a file another job still holds) or kept or moved it (it is no longer expired)."""
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
still = (
|
||||
job is not None
|
||||
and job.asset_id == asset_id
|
||||
and job.status == JobStatus.DONE
|
||||
and not job.kept
|
||||
and job.expires_at is not None
|
||||
and job.expires_at <= now
|
||||
)
|
||||
return job if still else None
|
||||
|
||||
|
||||
def _mark_warned(db: Session, job_id: int, expires_at: datetime) -> DownloadJob | None:
|
||||
"""Mark one file warned if it still goes when the pass saw it go — not deleted, kept, moved or
|
||||
re-timed since (each of those sets a fresh expiry, and its own warning to come) — and return it.
|
||||
The UPDATE's row lock holds it so until the commit."""
|
||||
hit = db.execute(
|
||||
update(DownloadJob)
|
||||
.where(
|
||||
DownloadJob.id == job_id,
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expiry_warned.is_(False),
|
||||
DownloadJob.expires_at == expires_at,
|
||||
)
|
||||
.values(expiry_warned=True)
|
||||
.returning(DownloadJob.id)
|
||||
.execution_options(synchronize_session=False)
|
||||
).scalar_one_or_none()
|
||||
return db.get(DownloadJob, job_id) if hit is not None else None
|
||||
|
||||
|
||||
def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
_notify_holders(db, asset, ntype)
|
||||
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
|
||||
def _on_disk(db: Session) -> int:
|
||||
"""The bytes the cache holds now (every asset whose file is on disk)."""
|
||||
return (
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
"""Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
|
||||
def _relock_evictable(db: Session, asset_id: int) -> tuple[MediaAsset | None, list[DownloadJob]]:
|
||||
"""Re-read an eviction candidate and its holders under their locks (the asset, then its jobs —
|
||||
the order the user routes take): between the pass's SELECT and here a holder may have kept it,
|
||||
or every holder deleted it."""
|
||||
asset = db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
holders = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.asset_id == asset_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return asset, list(holders)
|
||||
|
||||
|
||||
def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
|
||||
"""Take a file out of its user's library: tell them, release the job's hold on the asset (the
|
||||
file goes with the last holder), delete the row (its shares and links cascade)."""
|
||||
_notify(db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, ntype)
|
||||
service.release_asset(db, job)
|
||||
db.delete(job)
|
||||
|
||||
|
||||
def _sweep_orphan_files(db: Session, files_root: str) -> int:
|
||||
"""Delete media/sidecar files (and now-empty dirs) under FILES_ROOT that no `ready` asset
|
||||
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
|
||||
channel dir, or a poster from a superseded naming scheme). Skips the dot-prefixed system trees
|
||||
and only removes files older than `_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download
|
||||
is never deleted, and only files NOT owned by a surviving asset."""
|
||||
root = Path(download_root)
|
||||
channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
|
||||
row names at all. Skips the other dot-prefixed system trees and only removes files older than
|
||||
`_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
|
||||
NOT owned by a surviving asset."""
|
||||
root = Path(files_root)
|
||||
if not root.is_dir():
|
||||
return 0
|
||||
keep: set[Path] = set()
|
||||
# An upload's file is on disk from ingest on, whatever its state (a queued conversion's original,
|
||||
# a failed one kept for Retry), so every upload row keeps its file — not only a ready one.
|
||||
for rel in db.execute(
|
||||
select(MediaAsset.rel_path).where(
|
||||
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
||||
or_(MediaAsset.status == "ready", MediaAsset.source_kind == "upload"),
|
||||
MediaAsset.rel_path.is_not(None),
|
||||
)
|
||||
).scalars():
|
||||
media = (root / rel).resolve()
|
||||
@@ -99,7 +184,10 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
|
||||
unreclaimed = 0
|
||||
for entry in root.iterdir():
|
||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
||||
# .edits / .plex-* / .staging / .storyboards + any dotfile — but `.uploads` is swept: a
|
||||
# conversion stopped between placing the new file and the commit that points at it (or the
|
||||
# unlink of the original after it) leaves a file no row names, and nothing else finds it.
|
||||
if entry.name.startswith(".") and entry.name != _UPLOAD_TREE:
|
||||
continue
|
||||
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
||||
for f in files:
|
||||
@@ -143,74 +231,95 @@ def run_download_gc(db: Session) -> dict:
|
||||
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
||||
warned = expired = evicted = reaped = 0
|
||||
|
||||
# 1. Pre-expiry warnings.
|
||||
# 1. Pre-expiry warnings — one per file, the first run inside the grace window. Each mark is
|
||||
# conditional on the expiry the SELECT saw and committed on its own: a file deleted since is
|
||||
# passed over (a write to its gone row would fail the whole run), and one kept, moved or re-timed
|
||||
# since keeps its fresh expiry unwarned.
|
||||
warn_before = now + timedelta(days=grace_days)
|
||||
soon = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.gc_notified.is_(False),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at > now,
|
||||
MediaAsset.expires_at <= warn_before,
|
||||
)
|
||||
soon = db.execute(
|
||||
select(DownloadJob.id, DownloadJob.expires_at).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expiry_warned.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at > now,
|
||||
DownloadJob.expires_at <= warn_before,
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in soon:
|
||||
_notify_holders(db, asset, "download_expiring")
|
||||
asset.gc_notified = True
|
||||
warned += 1
|
||||
if soon:
|
||||
).all()
|
||||
db.rollback() # the pass-1 snapshot holds no locks; each mark takes its own below
|
||||
for job_id, expires_at in soon:
|
||||
job = _mark_warned(db, job_id, expires_at)
|
||||
if job is not None:
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
_notify(db, job, asset, "download_expiring")
|
||||
warned += 1
|
||||
db.commit()
|
||||
|
||||
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
||||
dead = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status.in_(("ready", "error")),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at <= now,
|
||||
)
|
||||
# 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
|
||||
# Each file is re-checked under its locks and committed on its own: the release unlinks the
|
||||
# file at once, so it must never act on what the SELECT saw a while ago.
|
||||
dead = db.execute(
|
||||
select(DownloadJob.id, DownloadJob.asset_id).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at <= now,
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in dead:
|
||||
_delete_asset(db, asset, "download_expired")
|
||||
expired += 1
|
||||
if dead:
|
||||
).all()
|
||||
db.rollback() # the pass-2 snapshot holds no locks; each file takes its own below
|
||||
for job_id, asset_id in dead:
|
||||
job = _relock_expired(db, job_id, asset_id, now)
|
||||
if job is not None:
|
||||
_drop_job(db, job, "download_expired")
|
||||
expired += 1
|
||||
db.commit()
|
||||
|
||||
# 3. LRU eviction over the total cache cap.
|
||||
# 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to
|
||||
# fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
|
||||
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
||||
if cap > 0:
|
||||
total = (
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.status == "ready"
|
||||
)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
if total > cap:
|
||||
candidates = (
|
||||
if _on_disk(db) > cap:
|
||||
kept_assets = select(DownloadJob.asset_id).where(
|
||||
DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
|
||||
)
|
||||
candidate_ids = (
|
||||
db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.status == "ready")
|
||||
select(MediaAsset.id)
|
||||
.where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.source_kind != "upload",
|
||||
MediaAsset.id.not_in(kept_assets),
|
||||
)
|
||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in candidates:
|
||||
if total <= cap:
|
||||
db.rollback()
|
||||
for asset_id in candidate_ids:
|
||||
# Counted afresh before each: a file deleted since (this one, an upload, a candidate
|
||||
# further down) has freed its space, and no other user's file goes for that space.
|
||||
if _on_disk(db) <= cap:
|
||||
break
|
||||
total -= asset.size_bytes or 0
|
||||
_delete_asset(db, asset, "download_evicted")
|
||||
# Locked and re-read like an expiry: since the SELECT a holder may have kept it,
|
||||
# every holder deleted it, or someone asked for it again (a fresh cache hit).
|
||||
asset, holders = _relock_evictable(db, asset_id)
|
||||
if (
|
||||
asset is None
|
||||
or asset.status != "ready"
|
||||
or any(j.kept for j in holders)
|
||||
or (asset.last_access_at is not None and asset.last_access_at > now)
|
||||
):
|
||||
db.commit()
|
||||
continue
|
||||
for job in holders:
|
||||
_notify(db, job, asset, "download_evicted")
|
||||
db.delete(job)
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
evicted += 1
|
||||
db.commit()
|
||||
db.commit()
|
||||
|
||||
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
||||
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
||||
@@ -232,9 +341,35 @@ def run_download_gc(db: Session) -> dict:
|
||||
if orphans:
|
||||
db.commit()
|
||||
|
||||
# 4b. Reap unheld uploads. An upload carries no expires_at, is never evicted and never re-enqueued,
|
||||
# so the passes above never free one that lost its last holder outside `release_asset` — a
|
||||
# conversion that finished after its job was cancelled, or one cancelled mid-run (the asset
|
||||
# was `downloading` then). Its file, ready or still the unconverted original, is at `rel_path`.
|
||||
unheld = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.source_kind == "upload",
|
||||
MediaAsset.ref_count == 0,
|
||||
MediaAsset.status != "downloading",
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in unheld:
|
||||
if asset.rel_path: # no holder is left to tell, so no notice — just the file and the row
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
reaped += 1
|
||||
if unheld:
|
||||
db.commit()
|
||||
|
||||
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
|
||||
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
|
||||
swept = _sweep_orphan_files(db, settings.download_root)
|
||||
swept = _sweep_orphan_files(db, settings.files_root)
|
||||
|
||||
# 6. A machine's token about to run out.
|
||||
tokens_warned = apitokens.warn_expiring(db)
|
||||
|
||||
result = {
|
||||
"warned": warned,
|
||||
@@ -242,6 +377,7 @@ def run_download_gc(db: Session) -> dict:
|
||||
"evicted": evicted,
|
||||
"reaped": reaped,
|
||||
"swept": swept,
|
||||
"tokens_warned": tokens_warned,
|
||||
}
|
||||
log.info("download_gc %s", result)
|
||||
return result
|
||||
|
||||
@@ -68,8 +68,8 @@ def _reset_asset_pending(asset_id: int) -> None:
|
||||
|
||||
|
||||
def _staging_dir(asset_id: int) -> Path:
|
||||
return Path(settings.download_root) / ".staging" / f"asset-{asset_id}"
|
||||
return Path(settings.files_root) / ".staging" / f"asset-{asset_id}"
|
||||
|
||||
|
||||
def _edit_staging_dir(asset_id: int) -> Path:
|
||||
return Path(settings.download_root) / ".staging" / f"edit-{asset_id}"
|
||||
return Path(settings.files_root) / ".staging" / f"edit-{asset_id}"
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
"""Public "share by link" helpers for the download center.
|
||||
|
||||
A DownloadLink is a capability URL: the unguessable `token` grants anyone read access to one
|
||||
download's file on the login-free `/watch/{token}` page. Password-protected links additionally
|
||||
require a short-lived signed **grant** on the file request — because a `<video src>` can't send a
|
||||
header, the password is exchanged once at `/unlock` for an HMAC grant appended to the file URL,
|
||||
so the raw password never rides in the media URL (or server logs).
|
||||
A DownloadLink is a capability URL: the unguessable token grants anyone read access to one
|
||||
download's file, or to the files in one of its owner's own folders (`folder_files`), on the
|
||||
login-free `/watch/{token}` page. So the token is never stored as written:
|
||||
the link is found by its SHA-256, and the owner's copy is Fernet-encrypted — a database copy alone
|
||||
opens nothing. Without that copy (no TOKEN_ENCRYPTION_KEY, or a key since changed) the link still
|
||||
opens; only its address cannot be shown again, and the owner makes a new link to share.
|
||||
|
||||
Password-protected links additionally require a short-lived signed **grant** on the file request —
|
||||
because a `<video src>` can't send a header, the password is exchanged once at `/unlock` for an HMAC
|
||||
grant appended to the file URL, so the raw password never rides in the media URL (or server logs).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
@@ -12,16 +17,71 @@ import hmac
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import security
|
||||
from app.config import settings
|
||||
from app.models import DownloadLink
|
||||
from app.downloads import storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
|
||||
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
|
||||
# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files.
|
||||
# MAX_LIST`; a test holds the two equal — this layer may not import that one).
|
||||
FOLDER_MAX = 500
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(24) # ~32 chars, 192 bits
|
||||
|
||||
|
||||
def seal(link: DownloadLink, token: str) -> None:
|
||||
"""Store `token` on `link` the only ways it is kept: its hash, and its encrypted copy."""
|
||||
link.token_hash = security.hash_token(token)
|
||||
link.token_enc = security.encrypt(token) if security.can_encrypt() else None
|
||||
|
||||
|
||||
def find(db: Session, token: str) -> DownloadLink | None:
|
||||
"""The link a `/watch` token names, by its hash."""
|
||||
return db.execute(
|
||||
select(DownloadLink).where(DownloadLink.token_hash == security.hash_token(token))
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def recover(link: DownloadLink) -> str | None:
|
||||
"""The link's token from its encrypted copy, or None when there is none this key can read."""
|
||||
return security.decrypt_optional(link.token_enc)
|
||||
|
||||
|
||||
def in_folder(folder: FileFolder) -> tuple:
|
||||
"""Which jobs a link to `folder` reaches: the done files in it NOW, the folder owner's. A file
|
||||
moved out stops opening through the link at its next request; one moved in starts."""
|
||||
return (
|
||||
DownloadJob.folder_id == folder.id,
|
||||
DownloadJob.user_id == folder.user_id,
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
)
|
||||
|
||||
|
||||
def folder_files(db: Session, folder: FileFolder) -> list[tuple[DownloadJob, MediaAsset]]:
|
||||
"""What a link to `folder` shows — its page lists exactly these, and its link card counts them:
|
||||
each file `in_folder` whose asset is ready and whose bytes are on disk (so the list never offers
|
||||
a dead row), newest first, at most `FOLDER_MAX`."""
|
||||
rows = db.execute(
|
||||
select(DownloadJob, MediaAsset)
|
||||
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||
.where(*in_folder(folder), MediaAsset.status == "ready")
|
||||
.order_by(DownloadJob.ready_at.desc().nulls_last(), DownloadJob.id.desc())
|
||||
.limit(FOLDER_MAX)
|
||||
).all()
|
||||
return [
|
||||
(job, asset)
|
||||
for job, asset in rows
|
||||
if storage.safe_abs_path(settings.files_root, asset.rel_path) is not None
|
||||
]
|
||||
|
||||
|
||||
def is_expired(link: DownloadLink) -> bool:
|
||||
return link.expires_at is not None and link.expires_at <= datetime.now(timezone.utc)
|
||||
|
||||
@@ -55,18 +115,22 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
|
||||
return False
|
||||
if exp < datetime.now(timezone.utc).timestamp():
|
||||
return False
|
||||
return hmac.compare_digest(sig, _sign(f"{token}.{exp}.{password_version}"))
|
||||
# As bytes: compare_digest refuses (TypeError, a 500) a str with anything non-ASCII in it, and
|
||||
# `sig` is whatever the URL carried.
|
||||
return hmac.compare_digest(sig.encode(), _sign(f"{token}.{exp}.{password_version}").encode())
|
||||
|
||||
|
||||
# --- serialization -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
||||
"""What the file owner sees when managing their links."""
|
||||
def owner_view(link: DownloadLink, base_url: str = "", token: str | None = None) -> dict:
|
||||
"""What the file owner sees when managing their links. `token` is the one just made (it is in
|
||||
hand even where no copy is kept); else it is read back, and `url` is None when it cannot be."""
|
||||
token = token or recover(link)
|
||||
return {
|
||||
"id": link.id,
|
||||
"token": link.token,
|
||||
"url": f"{base_url}/watch/{link.token}",
|
||||
"token": token,
|
||||
"url": f"{base_url}/watch/{token}" if token else None,
|
||||
"allow_download": link.allow_download,
|
||||
"has_password": link.password_hash is not None,
|
||||
"expires_at": link.expires_at.isoformat() if link.expires_at else None,
|
||||
|
||||
+37
-10
@@ -11,13 +11,12 @@ import html
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, DownloadLink, MediaAsset
|
||||
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||
|
||||
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
|
||||
|
||||
@@ -74,22 +73,25 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
|
||||
|
||||
def _watch_tags(token: str, db: Session) -> str | None:
|
||||
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
|
||||
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
|
||||
link = linksmod.find(db, token)
|
||||
if link is None or linksmod.is_expired(link):
|
||||
return None
|
||||
url = f"{settings.app_base}/watch/{token}"
|
||||
folder = link.folder_id is not None
|
||||
if link.password_hash:
|
||||
# Password-gated: don't leak the title/thumbnail in the unfurl.
|
||||
return "\n ".join(
|
||||
[
|
||||
"<title>Siftlode</title>",
|
||||
_tag("og:title", "Password-protected video"),
|
||||
_tag("og:title", "Password-protected " + ("folder" if folder else "video")),
|
||||
_tag("og:description", "Shared via Siftlode"),
|
||||
_tag("og:type", "video.other"),
|
||||
_tag("og:type", "website" if folder else "video.other"),
|
||||
_tag("og:url", url),
|
||||
_tag("og:site_name", "Siftlode"),
|
||||
]
|
||||
)
|
||||
if folder:
|
||||
return _folder_tags(db, link, url)
|
||||
job = db.get(DownloadJob, link.job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
||||
if job is None or asset is None or asset.status != "ready":
|
||||
@@ -135,7 +137,7 @@ def _watch_tags(token: str, db: Session) -> str | None:
|
||||
if image_dims is None and image.endswith("poster.jpg") and asset.poster_path:
|
||||
# safe_abs_path returns None if the poster file is missing (gc'd) or escapes the root —
|
||||
# guard it, else _jpeg_size(None) would raise an uncaught AttributeError on this public page.
|
||||
poster_abs = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
poster_abs = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||
image_dims = _jpeg_size(poster_abs) if poster_abs else None
|
||||
if image_dims:
|
||||
tags.append(_tag("og:image:width", str(image_dims[0])))
|
||||
@@ -147,13 +149,38 @@ def _watch_tags(token: str, db: Session) -> str | None:
|
||||
return "\n ".join(tags)
|
||||
|
||||
|
||||
def render_watch_html(token: str, index_html: Path, db: Session) -> str | None:
|
||||
"""Return index.html with the OG block replaced for this watch token, or None to serve the
|
||||
generic page unchanged (invalid/expired/not-ready link, or a template without the markers)."""
|
||||
def _folder_tags(db: Session, link: DownloadLink, url: str) -> str | None:
|
||||
"""A folder link's card: the folder's name and how many files it holds — never their names,
|
||||
which the page shows only to whoever opens it."""
|
||||
folder = db.get(FileFolder, link.folder_id)
|
||||
if folder is None or not folder.name:
|
||||
return None
|
||||
count = len(linksmod.folder_files(db, folder)) # the page's own list, so the two agree
|
||||
title = folder.name.strip()
|
||||
about = f"{count} file{'' if count == 1 else 's'} · Shared via Siftlode"
|
||||
return "\n ".join(
|
||||
[
|
||||
f"<title>{html.escape(title)}</title>",
|
||||
_tag("og:title", title),
|
||||
_tag("og:description", about),
|
||||
_tag("og:type", "website"),
|
||||
_tag("og:url", url),
|
||||
_tag("og:site_name", "Siftlode"),
|
||||
_tag("twitter:title", title, attr="name"),
|
||||
_tag("twitter:description", about, attr="name"),
|
||||
_tag("twitter:card", "summary", attr="name"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def render_watch_html(token: str, page_html: Path, db: Session) -> str | None:
|
||||
"""Return the watch page (watch.html) with the OG block replaced for this token, or None to
|
||||
serve the generic page unchanged (invalid/expired/not-ready link, or a template without the
|
||||
markers)."""
|
||||
tags = _watch_tags(token, db)
|
||||
if tags is None:
|
||||
return None
|
||||
text = index_html.read_text(encoding="utf-8")
|
||||
text = page_html.read_text(encoding="utf-8")
|
||||
if "<!--OG:START-->" not in text:
|
||||
return None
|
||||
# A function replacement avoids re.sub interpreting backslashes/group refs in `tags`.
|
||||
|
||||
@@ -3,17 +3,20 @@
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
from yt_dlp.utils import download_range_func
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import adapters, credentials, formats, mediakind, probe, quota, service, storage
|
||||
from app.downloads import adapters, credentials, formats, mediakind, playable, probe, quota
|
||||
from app.downloads import service, storage
|
||||
from app.downloads import urlguard
|
||||
from app.downloads.jobdb import (
|
||||
_edit_staging_dir,
|
||||
@@ -25,7 +28,9 @@ from app.downloads.jobdb import (
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem
|
||||
from app.plex import lookups as plex_lookups
|
||||
from app.plex import paths as plex_paths
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
|
||||
@@ -36,6 +41,15 @@ class _Aborted(Exception):
|
||||
"""Raised inside the progress hook when the job was paused/canceled mid-download."""
|
||||
|
||||
|
||||
class _ClipFetchFailed(Exception):
|
||||
"""A clip's YouTube range fetch failed: its message and the download catalog's diagnosis, so a
|
||||
clip on YouTube's wall says what a download of the same video would."""
|
||||
|
||||
def __init__(self, msg: str, diagnosis: dlerrors.Diagnosis):
|
||||
super().__init__(msg)
|
||||
self.diagnosis = diagnosis
|
||||
|
||||
|
||||
class _OwnerRefused(Exception):
|
||||
"""A refusal that belongs to THIS job's owner, not to the source.
|
||||
|
||||
@@ -58,6 +72,10 @@ _DB_WRITE_INTERVAL_S = 1.0
|
||||
_CANCEL_POLL_INTERVAL_S = 1.0
|
||||
|
||||
_THUMB_SUFFIXES = (".jpg", ".jpeg", ".png", ".webp")
|
||||
# The job phase while a file is brought into the phone-safe shape (`_ensure_phone_safe`) for a
|
||||
# download or an upload. Its own word, not yt-dlp's "processing": it reports ffmpeg's real progress,
|
||||
# so the queue row shows a % for it, where yt-dlp's post-steps only get an indeterminate pulse.
|
||||
CONVERTING = "converting"
|
||||
# yt-dlp's working files in the staging dir. None of them is ever the finished media, but several
|
||||
# CAN be the biggest file there (a video-only per-format stream, an abandoned fragment run), so the
|
||||
# staging fallback must exclude them by name — picking one would store e.g. a silent video-only
|
||||
@@ -73,20 +91,28 @@ _YTDLP_TEMP_RE = re.compile(
|
||||
|
||||
def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *, kind: str) -> None:
|
||||
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
||||
silently while scheduler jobs and playlist-push already notified (R11 feedback symmetry). A
|
||||
silently while scheduler jobs and playlist-push already notified (feedback symmetry). A
|
||||
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
||||
scheduler._notify_done). `kind` ("download"/"edit") lets the inbox pick accurate copy; the
|
||||
scheduler._notify_done). `kind` ("download"/"edit"/"upload") lets the inbox pick accurate copy; the
|
||||
English title here is only a fallback for an unknown type. `user_id`/`title` are captured by the
|
||||
caller BEFORE its commit, so reading them here needs no post-commit refresh of an expired row."""
|
||||
caller BEFORE its commit; only the folder is read here, once, for the notice's "Open"."""
|
||||
if user_id is None:
|
||||
return
|
||||
try:
|
||||
# The notice's "Open" lands on the file in its folder: both ride along (a folder the file
|
||||
# has left since is the page's to notice — it then looks among every file).
|
||||
job = db.get(DownloadJob, job_id)
|
||||
create_notification(
|
||||
db,
|
||||
user_id=user_id,
|
||||
type="download_done",
|
||||
title=f"Ready to watch: {title}" if title else "Ready to watch",
|
||||
data={"title": title, "kind": kind},
|
||||
data={
|
||||
"title": title,
|
||||
"kind": kind,
|
||||
"job_id": job_id,
|
||||
"folder_id": job.folder_id if job else None,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
@@ -208,7 +234,7 @@ def _cancel_poll(job_id: int, interval_s: float = _CANCEL_POLL_INTERVAL_S, *, no
|
||||
|
||||
def cancelled() -> bool:
|
||||
if not state["cancelled"] and allow():
|
||||
state["cancelled"] = _job_status(job_id) not in ("running", None)
|
||||
state["cancelled"] = _job_status(job_id) not in (JobStatus.RUNNING, None)
|
||||
return state["cancelled"]
|
||||
|
||||
return cancelled
|
||||
@@ -219,7 +245,7 @@ def _ffmpeg_progress_writer(job_id: int, phase: str, *, now=time.monotonic):
|
||||
when the whole-percent value changed (the bar shows integers), plus a separate once-per-interval
|
||||
heartbeat while the percent is stuck (so a slow re-encode isn't mistaken for a stall)."""
|
||||
# Progress writes have their OWN 1/interval budget so a heartbeat can never delay a real write (the
|
||||
# round-1 concern). The heartbeat, in turn, is gated on `last_touch` — the last DB write OR beat —
|
||||
# original concern). The heartbeat, in turn, is gated on `last_touch` — the last DB write OR beat —
|
||||
# so a progress write (which already refreshes updated_at) suppresses a redundant heartbeat that
|
||||
# same interval, keeping active downloads at ~1 write/interval, not two.
|
||||
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S, now=now)
|
||||
@@ -328,7 +354,7 @@ def _media_info(info: dict) -> dict:
|
||||
|
||||
|
||||
def _extract_with_fallback(
|
||||
job_id, asset_id, url, spec, staging, adapter, cookiefile=None, max_bytes=None
|
||||
job_id, asset_id, url, spec, staging, adapter, cookiefile=None, max_bytes=None, section=None
|
||||
):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
@@ -341,10 +367,17 @@ def _extract_with_fallback(
|
||||
(adapters.POST_*_MESSAGE) instead of letting yt-dlp guess, and a failed page read fails the
|
||||
download as itself.
|
||||
|
||||
`cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated.
|
||||
`cookiefile` is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile` (a clip: `resolve_clip_cookiefile`); passed to every attempt so
|
||||
the fallback stays authenticated.
|
||||
|
||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt."""
|
||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt. A
|
||||
`section` fetch goes through yt-dlp's ffmpeg downloader, which does not read it: a clip holds
|
||||
its own output to the cap (`_process_clip`).
|
||||
|
||||
`section` = (start_s, end_s) fetches only that range (an ad-hoc clip, see downloads/clip): the
|
||||
range is stream-copied, and its first frame is the marked one (measured), so the clip's own
|
||||
re-encode starts at 0 instead of re-fetching keyframes at the cut."""
|
||||
client_sets = adapter.client_sets(authed=cookiefile is not None)
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
# Resolving may read a page first — network time the stall watchdog must not see as a stall.
|
||||
@@ -370,6 +403,9 @@ def _extract_with_fallback(
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
if section is not None:
|
||||
opts["download_ranges"] = download_range_func([], [section])
|
||||
opts["force_keyframes_at_cuts"] = False
|
||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||
# multi-client-set fallback would otherwise cross the stall timeout with a stale updated_at.
|
||||
# (Per-ATTEMPT only; a single extract_info that itself runs past the stall timeout is still
|
||||
@@ -406,58 +442,58 @@ def _extract_with_fallback(
|
||||
raise last_exc # unreachable (loop either returns or raises)
|
||||
|
||||
|
||||
def _download_settings() -> tuple[str, int]:
|
||||
"""Layout + retention days from the DB config (admin-overridable on the Configuration page),
|
||||
falling back to the env defaults. Read here, not from `settings`, so an admin edit takes effect."""
|
||||
def _download_layout() -> str:
|
||||
"""The on-disk layout from the DB config (admin-overridable on the Configuration page), falling
|
||||
back to the env default. Read here, not from `settings`, so an admin edit takes effect. (How long
|
||||
a file stays is its folder's, stamped on the job when it is done — `folders.done_expiry`.)"""
|
||||
with SessionLocal() as db:
|
||||
return (
|
||||
sysconfig.get_str(db, "download_layout"),
|
||||
sysconfig.get_int(db, "download_retention_days"),
|
||||
)
|
||||
return sysconfig.get_str(db, "download_layout")
|
||||
|
||||
|
||||
def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict) -> Path:
|
||||
"""Make an mp4-compat-profile download play in every browser: H.264 + AAC/MP3 in mp4. If the
|
||||
download (whatever the source shipped) isn't already that, re-encode the offending stream(s) —
|
||||
or just remux when the codecs are fine but the container isn't. **Scope:** only the mp4 compat
|
||||
profile is normalized; an explicit `vcodec` or a non-mp4 custom profile is the user opting out
|
||||
of the guarantee and is left as-is (so a `webm` profile can still ship VP9/Opus). Returns the
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk.
|
||||
def _ensure_phone_safe(
|
||||
job_id: int,
|
||||
produced: Path,
|
||||
out_dir: Path,
|
||||
phase: str,
|
||||
total_s: int | None = None,
|
||||
*,
|
||||
keep_source: bool = False,
|
||||
) -> Path:
|
||||
"""Bring a file that is about to become a ready library asset into the phone-safe shape
|
||||
(`playable`), writing any converted copy into `out_dir`. Returns the file to store — `produced`
|
||||
itself when it already is that shape, or isn't media at all — and removes `produced` once a
|
||||
converted copy exists, unless `keep_source` (an upload's original, which must outlive the
|
||||
conversion until the database points at the new file). The job shows `phase` with real ffmpeg
|
||||
progress while it converts.
|
||||
|
||||
An audio download (mode "a") is never touched: yt-dlp already extracted it to .mp3 / .m4a, which
|
||||
every browser plays as audio/mpeg / audio/mp4. Remuxing it into .mp4 served an mp3 as video/mp4,
|
||||
which iPad's WebKit refuses to play (prod job 268)."""
|
||||
if spec.get("mode") == "a" or spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
Every ready path calls this — a download, an upload's worker conversion and an editor clip — so
|
||||
a share link can only ever serve the one shape every phone and browser plays."""
|
||||
p = playable.plan(probe.probe_json(produced), produced.suffix)
|
||||
if p is None or not p.needs_change:
|
||||
return produced
|
||||
vcodec, acodec = editmod.probe_codecs(produced)
|
||||
is_mp4 = produced.suffix.lower() == ".mp4"
|
||||
needs, reenc_v, reenc_a = editmod.browser_transcode_flags(vcodec, acodec, is_mp4)
|
||||
if not needs:
|
||||
return produced
|
||||
|
||||
dest = produced.with_name(produced.stem + ".browser.mp4")
|
||||
action = "re-encode" if (reenc_v or reenc_a) else "remux"
|
||||
dest = out_dir / f"{produced.stem}.safe.{p.ext}"
|
||||
action = "re-encode" if (p.reencode_video or p.reencode_audio) else "remux"
|
||||
log.info(
|
||||
"job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4
|
||||
"job %s: %s (video=%s audio=%s) -> phone-safe .%s",
|
||||
job_id,
|
||||
action,
|
||||
p.reencode_video,
|
||||
p.reencode_audio,
|
||||
p.ext,
|
||||
extra=logs.event("download.phone_safe", job_id=job_id, action=action, ext=p.ext),
|
||||
)
|
||||
_set_job(job_id, phase="processing", progress=0, speed_bps=None, eta_s=None)
|
||||
# Prefer the info duration; probe the file when the extractor didn't provide one, so a long
|
||||
# re-encode still shows real progress instead of a frozen 0%.
|
||||
total_s = int(info["duration"]) if info.get("duration") else editmod.probe_duration(produced)
|
||||
_set_job(job_id, phase=phase, progress=0, speed_bps=None, eta_s=None)
|
||||
# A re-encode with no known length would hold a frozen 0%; probe the file for a denominator.
|
||||
editmod.run_ffmpeg(
|
||||
editmod.build_browser_safe_cmd(
|
||||
produced, dest, reenc_v, reenc_a, threads=editmod.encode_thread_cap()
|
||||
),
|
||||
total_s=total_s,
|
||||
on_progress=_ffmpeg_progress_writer(job_id, "processing"),
|
||||
playable.build_cmd(produced, dest, p, threads=editmod.encode_thread_cap()),
|
||||
total_s=total_s or editmod.probe_duration(produced),
|
||||
on_progress=_ffmpeg_progress_writer(job_id, phase),
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
)
|
||||
produced.unlink(missing_ok=True)
|
||||
# Keep the stored asset metadata truthful about what's now on disk.
|
||||
if reenc_v:
|
||||
info["vcodec"] = "h264"
|
||||
if reenc_a:
|
||||
info["acodec"] = "aac"
|
||||
if not dest.exists():
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
if not keep_source:
|
||||
produced.unlink(missing_ok=True)
|
||||
return dest
|
||||
|
||||
|
||||
@@ -483,7 +519,7 @@ def _download(
|
||||
urlguard.check_url(url)
|
||||
except urlguard.BlockedUrl as e:
|
||||
raise StatedFailure(f"Refusing to download from this address ({e.reason}).") from e
|
||||
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# Attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
remaining_bytes: int | None = None
|
||||
with SessionLocal() as cdb:
|
||||
@@ -495,8 +531,8 @@ def _download(
|
||||
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
|
||||
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a
|
||||
# single job could previously blow through the cap by any amount. Handing the remainder
|
||||
# to yt-dlp as `max_filesize` makes the format selector refuse an oversized file up
|
||||
# front instead.
|
||||
# to yt-dlp as `max_filesize` makes it refuse an oversized file (its HTTP downloader
|
||||
# checks the announced size) instead.
|
||||
remaining_bytes = quota.remaining_bytes(cdb, user_id)
|
||||
# …and zero is a REFUSAL, not "no limit". Earlier jobs of this user may have filled the cap
|
||||
# while this one sat in the queue, and 0 is not an expressible yt-dlp `max_filesize`, so
|
||||
@@ -505,8 +541,8 @@ def _download(
|
||||
# free space, which is what the `quota_exceeded` category tells them.
|
||||
if remaining_bytes == 0:
|
||||
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
|
||||
info = _media_info(
|
||||
_extract_with_fallback(
|
||||
with formats.watching_size_cap() as over_cap:
|
||||
fetched = _extract_with_fallback(
|
||||
job_id,
|
||||
asset_id,
|
||||
url,
|
||||
@@ -516,13 +552,30 @@ def _download(
|
||||
cookiefile=str(cookiefile) if cookiefile else None,
|
||||
max_bytes=remaining_bytes,
|
||||
)
|
||||
)
|
||||
info = _media_info(dict(fetched))
|
||||
# yt-dlp refused a file larger than what is left: it returns with no file, which would
|
||||
# otherwise read as a processing fault. The owner's storage, so the owner's refusal.
|
||||
if over_cap[0]:
|
||||
raise _OwnerRefused(dlerrors.DOWNLOAD_TOO_BIG_MESSAGE)
|
||||
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
||||
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||
|
||||
produced = _produced_file(info, staging)
|
||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||
produced = _ensure_phone_safe(
|
||||
job_id,
|
||||
produced,
|
||||
staging,
|
||||
CONVERTING,
|
||||
int(info["duration"]) if info.get("duration") else None,
|
||||
)
|
||||
# Held to the cap by its real size, as a clip is: yt-dlp's `max_filesize` only stops a plain
|
||||
# HTTP file that announces its size, and YouTube's fragmented formats never meet it
|
||||
# (measured: 718 MB fetched with 1 MB left). Read again — other jobs may have landed.
|
||||
with SessionLocal() as qdb:
|
||||
room = quota.remaining_bytes(qdb, user_id)
|
||||
if room is not None and produced.stat().st_size > room:
|
||||
raise _OwnerRefused(dlerrors.DOWNLOAD_TOO_BIG_MESSAGE)
|
||||
ext = produced.suffix.lstrip(".").lower()
|
||||
# Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's
|
||||
# `info` dict, whose fields are absent for some sources (Facebook among them): deriving
|
||||
@@ -541,19 +594,18 @@ def _download(
|
||||
thumbnail_url=info.get("thumbnail"),
|
||||
)
|
||||
|
||||
layout, retention_days = _download_settings()
|
||||
layout = _download_layout()
|
||||
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
|
||||
thumb = _find_thumbnail(staging, produced)
|
||||
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
|
||||
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
|
||||
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
|
||||
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
|
||||
storage.place_file(produced, settings.download_root, rel)
|
||||
nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, poster_jpg)
|
||||
storage.place_file(produced, settings.files_root, rel)
|
||||
nfo_ok = storage.write_sidecars(settings.files_root, rel, meta, poster_jpg)
|
||||
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
size = final.stat().st_size if final.exists() else None
|
||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
@@ -579,12 +631,11 @@ def _download(
|
||||
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
|
||||
# and can't be relied on cross-origin. write_sidecars already wrote <base>.jpg from
|
||||
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||
asset.source_webpage_url = info.get("webpage_url")
|
||||
asset.nfo_written = nfo_ok
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = expires
|
||||
job = db.get(DownloadJob, job_id)
|
||||
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
||||
owner_id = job.user_id if job else None
|
||||
@@ -597,7 +648,7 @@ def _download(
|
||||
}
|
||||
if job and not job.display_name and meta.title:
|
||||
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
||||
fields["display_name"] = meta.title[:255]
|
||||
fields["display_name"] = meta.title[:DISPLAY_TEXT_MAX]
|
||||
# Atomic terminal write: finalizes ONLY while the row is still running, so a child that
|
||||
# finishes just after a route cancel/pause can't resurrect it (see finalize_if_running).
|
||||
finalized = service.finalize_if_running(db, job_id, **fields)
|
||||
@@ -722,9 +773,19 @@ def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseExcep
|
||||
)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None = None) -> None:
|
||||
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
|
||||
where its line and its event code live. `exc` is the exception when there was one.
|
||||
def _fail_edit(
|
||||
job_id: int,
|
||||
asset_id: int,
|
||||
msg: str,
|
||||
exc: BaseException | None = None,
|
||||
*,
|
||||
what: str = "edit",
|
||||
diagnosis: dlerrors.Diagnosis | None = None,
|
||||
) -> None:
|
||||
"""Fail a local ffmpeg job — an edit, a clip, or an upload's conversion (`what="upload"`) — and
|
||||
say so ONCE: every way one fails comes through here, so this is where its line and its event code
|
||||
(`<what>.job.failed`) live. `exc` is the exception when there was one; `diagnosis` replaces the
|
||||
plain category when the failure was a fetch the download catalog can name (a clip's).
|
||||
|
||||
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
|
||||
Logged after the write, as on the download error path."""
|
||||
@@ -739,19 +800,25 @@ def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None =
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
error_code=diagnosis.code
|
||||
if diagnosis
|
||||
else dlerrors.classify_conversion(msg)
|
||||
if what == "upload"
|
||||
else dlerrors.classify(msg),
|
||||
error_detail=diagnosis.detail if diagnosis else None,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
||||
# built from the channel and the title.
|
||||
log.warning(
|
||||
"edit job %s failed: %s",
|
||||
"%s job %s failed: %s",
|
||||
what,
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra={
|
||||
**logs.event(
|
||||
"edit.job.failed",
|
||||
f"{what}.job.failed",
|
||||
job_id=job_id,
|
||||
asset_id=asset_id,
|
||||
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
||||
@@ -777,7 +844,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
if not src_ready or not src_rel:
|
||||
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
||||
return
|
||||
src_path = storage.abs_path(settings.download_root, src_rel)
|
||||
src_path = storage.abs_path(settings.files_root, src_rel)
|
||||
if not src_path.exists():
|
||||
_fail_edit(job_id, asset_id, "The source file is missing.")
|
||||
return
|
||||
@@ -811,13 +878,24 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
)
|
||||
if not dest_staging.exists():
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
# A fast trim stream-copies the source's codecs and container, so the clip is only as
|
||||
# playable as its source: hold it to the same rule as every other ready file.
|
||||
dest_staging = _ensure_phone_safe(job_id, dest_staging, staging, "editing", total_s)
|
||||
out_ext = dest_staging.suffix.lstrip(".").lower() or out_ext
|
||||
# New storage on the owner's footprint, held to what is left by its real size, as a
|
||||
# download and a player clip are.
|
||||
if user_id is not None:
|
||||
with SessionLocal() as qdb:
|
||||
room = quota.remaining_bytes(qdb, user_id)
|
||||
if room is not None and dest_staging.stat().st_size > room:
|
||||
raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||
|
||||
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||
storage.place_file(dest_staging, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
storage.place_file(dest_staging, settings.files_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
size = final.stat().st_size if final.exists() else None
|
||||
_, retention_days = _download_settings()
|
||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
|
||||
_, _, out_w, out_h, _ = probe.probe_media(final)
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
@@ -827,14 +905,15 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
asset.size_bytes = size
|
||||
asset.container = out_ext
|
||||
asset.duration_s = total_s or asset.duration_s
|
||||
if crop:
|
||||
if out_w and out_h:
|
||||
asset.width, asset.height = out_w, out_h
|
||||
elif crop:
|
||||
asset.width = edit_spec["crop"]["w"]
|
||||
asset.height = edit_spec["crop"]["h"]
|
||||
else:
|
||||
asset.width, asset.height = src_w, src_h
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = expires
|
||||
job = db.get(DownloadJob, job_id)
|
||||
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
|
||||
owner_id = job.user_id if job else None
|
||||
@@ -867,3 +946,388 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _clip_room(user_id: int) -> int | None:
|
||||
"""What the clip's owner may still store (`quota.remaining_bytes`), refusing at zero — see
|
||||
`_download`: zero is a refusal, not "no limit"."""
|
||||
with SessionLocal() as db:
|
||||
remaining = quota.remaining_bytes(db, user_id)
|
||||
if remaining == 0:
|
||||
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
|
||||
return remaining
|
||||
|
||||
|
||||
def _clip_source(
|
||||
job_id: int, asset_id: int, spec: dict, staging: Path, remaining: int | None
|
||||
) -> tuple[Path, float, str | None]:
|
||||
"""The file a clip is cut from, where in it the clip starts, and the source's title when only
|
||||
the fetch could tell it. A YouTube clip fetches ONLY its range into `staging` (so it starts at
|
||||
0 there); a Plex clip reads the library file in place."""
|
||||
if spec["source"] == "youtube":
|
||||
url = service.source_url("youtube", spec["ref"])
|
||||
cookiefile: Path | None = None
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
# Signed in when the user has a YouTube cookie, as their downloads are: YouTube gates
|
||||
# single videos signed out ("not a bot"), and the clip is theirs alone.
|
||||
if job and asset:
|
||||
cookiefile = credentials.resolve_clip_cookiefile(
|
||||
db, job.user_id, url, asset, spec, staging
|
||||
)
|
||||
_set_job(job_id, phase="downloading", progress=0)
|
||||
try:
|
||||
with formats.watching_size_cap() as over_cap:
|
||||
info = _extract_with_fallback(
|
||||
job_id,
|
||||
asset_id,
|
||||
url,
|
||||
formats.normalize(clipmod.YOUTUBE_FETCH_SPEC),
|
||||
staging,
|
||||
adapters.resolve_adapter("youtube", spec["ref"]),
|
||||
cookiefile=str(cookiefile) if cookiefile else None,
|
||||
max_bytes=remaining,
|
||||
section=(spec["start_s"], spec["end_s"]),
|
||||
)
|
||||
# Should yt-dlp's own cap ever refuse the range (see `_download`): the owner's storage.
|
||||
if over_cap[0]:
|
||||
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||
except (_Aborted, _OwnerRefused):
|
||||
raise
|
||||
except Exception as exc: # re-raised below with the download catalog's diagnosis
|
||||
msg = str(exc)[:500]
|
||||
raise _ClipFetchFailed(
|
||||
msg, dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
|
||||
) from exc
|
||||
finally:
|
||||
# Never leave a plaintext session cookie on disk past the fetch (see `_download`).
|
||||
if cookiefile is not None:
|
||||
cookiefile.unlink(missing_ok=True)
|
||||
info = _media_info(dict(info))
|
||||
title = normalize_title(adapters.resolve_adapter("youtube", spec["ref"]).title(info))
|
||||
return _produced_file(info, staging), 0.0, title
|
||||
with SessionLocal() as db:
|
||||
item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"])
|
||||
src = plex_paths.local_media_path(db, item.file_path) if item else None
|
||||
if src is None:
|
||||
raise StatedFailure("The library file is not reachable from here.")
|
||||
return src, spec["start_s"], None
|
||||
|
||||
|
||||
def _clip_crop(spec: dict, data: dict) -> str:
|
||||
"""The clip's crop filter, from its frame and the picture the file really holds ("" for none)."""
|
||||
frame = spec.get("crop")
|
||||
if not frame:
|
||||
return ""
|
||||
dar = clipmod.display_aspect(data)
|
||||
if dar is None:
|
||||
raise StatedFailure("The picture's size could not be read, so the crop cannot be placed.")
|
||||
crop = clipmod.crop_filter(frame, dar)
|
||||
if crop is None:
|
||||
raise StatedFailure("The crop frame lies outside the picture.")
|
||||
return crop
|
||||
|
||||
|
||||
def _process_clip(job_id: int, asset_id: int) -> None:
|
||||
"""Cut an ad-hoc clip (downloads/clip): fetch or open its source, ONE re-encode to the clip
|
||||
shape, then into the user's library beside their edits."""
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None:
|
||||
return
|
||||
spec = dict(job.edit_spec or {})
|
||||
user_id = job.user_id
|
||||
staging = _edit_staging_dir(asset_id)
|
||||
try:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
# Refused before the fetch and the encode when the owner's storage is already full.
|
||||
remaining = _clip_room(user_id)
|
||||
src, start, fetched_title = _clip_source(job_id, asset_id, spec, staging, remaining)
|
||||
data = probe.probe_json(src)
|
||||
if not data:
|
||||
raise StatedFailure("The source file could not be read.")
|
||||
audio_track = spec.get("audio_track")
|
||||
tracks = clipmod.audio_track_count(data)
|
||||
if tracks == 0:
|
||||
# A silent source: the player still names its track 0, and the clip is silent too.
|
||||
audio_track = None
|
||||
elif audio_track is not None and audio_track >= tracks:
|
||||
raise StatedFailure("The chosen audio track is not in the file.")
|
||||
crop = _clip_crop(spec, data)
|
||||
total_s = clipmod.duration(spec)
|
||||
out = staging / "clip.mp4"
|
||||
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
||||
editmod.run_ffmpeg(
|
||||
clipmod.build_cmd(
|
||||
src,
|
||||
out,
|
||||
start,
|
||||
total_s,
|
||||
audio_track=audio_track,
|
||||
hdr=clipmod.is_hdr(data),
|
||||
threads=editmod.encode_thread_cap(),
|
||||
crop=crop,
|
||||
),
|
||||
total_s,
|
||||
on_progress=_ffmpeg_progress_writer(job_id, "editing"),
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
)
|
||||
if not out.exists():
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
# The phone-safe shape by construction; still held to the rule every ready file meets.
|
||||
out = _ensure_phone_safe(job_id, out, staging, "editing", round(total_s))
|
||||
# Held to the cap by its real size: a range fetch passes yt-dlp's `max_filesize` to its
|
||||
# ffmpeg downloader, which never reads it, and a Plex clip fetches nothing at all. Read
|
||||
# again, since other jobs of this owner may have landed during the encode.
|
||||
remaining = _clip_room(user_id)
|
||||
if remaining is not None and out.stat().st_size > remaining:
|
||||
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||
ext = out.suffix.lstrip(".").lower() or "mp4"
|
||||
rel = storage.edit_rel_path(user_id, asset_id, ext)
|
||||
storage.place_file(out, settings.files_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
vcodec, acodec, width, height, _ = probe.probe_media(final)
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "ready"
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = final.stat().st_size if final.exists() else None
|
||||
asset.container = ext
|
||||
asset.vcodec, asset.acodec, asset.width, asset.height = (
|
||||
vcodec,
|
||||
acodec,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
asset.duration_s = round(total_s)
|
||||
asset.media_kind = mediakind.derive(vcodec, acodec)
|
||||
# The clip's own frame as its poster: the link preview shows the moment, and a Plex
|
||||
# item has no public thumbnail at all.
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
fields: dict = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
||||
# A video outside the catalog was queued as "Clip (clip)": the fetch has its title now.
|
||||
placeholder = service.clip_title(None, None)
|
||||
if asset and fetched_title and asset.title == placeholder:
|
||||
named = service.clip_title(fetched_title, None)
|
||||
asset.title = named
|
||||
if job_row := db.get(DownloadJob, job_id):
|
||||
if job_row.display_name == placeholder:
|
||||
fields["display_name"] = named[:DISPLAY_TEXT_MAX]
|
||||
title = asset.title if asset else None
|
||||
finalized = service.finalize_if_running(db, job_id, **fields)
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, user_id, title, job_id, kind="edit")
|
||||
log.info(
|
||||
"clip job %s done: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event(
|
||||
"clip.job.done", job_id=job_id, asset_id=asset_id, source=spec["source"]
|
||||
),
|
||||
)
|
||||
except (_Aborted, editmod.EditAborted):
|
||||
# Paused/canceled mid-fetch or mid-encode: free the asset for a later retry.
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info(
|
||||
"clip job %s aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("clip.job.aborted", job_id=job_id),
|
||||
)
|
||||
except _OwnerRefused as exc:
|
||||
# Quota full: the job fails, the asset goes back to pending for a Retry once there is room
|
||||
# (the download's handling; see `_download`).
|
||||
msg = str(exc)[:500]
|
||||
with SessionLocal() as db:
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
error_detail=None,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info(
|
||||
"clip job %s refused for its owner: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event("clip.job.refused", job_id=job_id),
|
||||
)
|
||||
except _ClipFetchFailed as exc:
|
||||
# silent-ok: not silent — `_fail_edit` is the report, with the fetch's diagnosis.
|
||||
_fail_edit(job_id, asset_id, str(exc), exc, what="clip", diagnosis=exc.diagnosis)
|
||||
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="clip")
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _check_converted_fits(owner_id: int | None, old_size: int, new_size: int) -> None:
|
||||
"""Refuse a converted upload that would push its owner over the storage cap: the upload was
|
||||
charged at its original size, and a re-encode can come out larger. The original stays, counted,
|
||||
for the owner to delete or keep."""
|
||||
if owner_id is None:
|
||||
return
|
||||
with SessionLocal() as db:
|
||||
if not quota.fits_after_conversion(db, owner_id, old_size, new_size):
|
||||
raise StatedFailure(dlerrors.CONVERTED_TOO_BIG_MESSAGE)
|
||||
|
||||
|
||||
def _unplace(placed: Path | None) -> None:
|
||||
"""Remove a converted upload that was placed at its new rel but never committed: the row still
|
||||
names the original, so nothing else would ever find this file."""
|
||||
if placed is not None:
|
||||
placed.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _conversion_landed(asset_id: int, new_rel: str) -> bool | None:
|
||||
"""After the commit that makes a converted upload ready RAISED: did it land anyway? A commit can
|
||||
reach the server and lose its answer (the connection drops after COMMIT). True/False by what the
|
||||
row now says; None when the row cannot be read either."""
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
return asset is not None and asset.status == "ready" and asset.rel_path == new_rel
|
||||
except Exception: # noqa: BLE001 — "cannot tell" is the answer, the caller acts on it
|
||||
# silent-ok: the caller re-raises the commit's own error, which `_fail_edit` reports.
|
||||
return None
|
||||
|
||||
|
||||
def _drop_original(job_id: int, src: Path) -> None:
|
||||
"""Remove an upload's original once the converted file is committed in its place. Best effort:
|
||||
the job is done and its file is ready, so a failure here must not fail it — the original is
|
||||
named by no row now, and the GC's `.uploads` sweep takes it."""
|
||||
try:
|
||||
src.unlink(missing_ok=True)
|
||||
except OSError as exc:
|
||||
log.warning(
|
||||
"upload job %s: original not removed, left to the GC sweep: %s",
|
||||
job_id,
|
||||
exc,
|
||||
extra=logs.event("upload.original.unlink_failed", job_id=job_id),
|
||||
)
|
||||
|
||||
|
||||
def _free_unheld_upload(asset_id: int) -> bool:
|
||||
"""Free an upload no job holds any more — cancelled or deleted while its conversion ran, when
|
||||
`release_asset` had to leave it because the conversion still read its file. Once that has
|
||||
stopped, free it as the release would have; left alone, its file would sit on disk counted by no
|
||||
owner's footprint until the GC's upload pass. False when a job still holds it (paused)."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset is None or asset.ref_count != 0:
|
||||
return False
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
"""Convert a queued upload into the phone-safe shape (`playable`) and make it ready.
|
||||
|
||||
The original sits at the asset's `rel_path` (see `upload`); the converted file replaces it there
|
||||
under the new extension, and only then is the original removed — so a failed or stopped
|
||||
conversion leaves the original in place for Retry, and a delete still finds it by `rel_path`.
|
||||
A converted file placed but never committed is removed again on the way out. When the commit's
|
||||
own outcome is unknown neither file is removed: the row names one of them, and the GC's `.uploads`
|
||||
sweep takes the one it does not — as it does an original whose unlink after the commit failed."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
old_rel = asset.rel_path if asset else None
|
||||
old_ext = asset.container if asset else None
|
||||
old_size = (asset.size_bytes or 0) if asset else 0
|
||||
duration = asset.duration_s if asset else None
|
||||
job = db.get(DownloadJob, job_id)
|
||||
owner_id = job.user_id if job else None
|
||||
src = storage.safe_abs_path(settings.files_root, old_rel) if old_rel else None
|
||||
if old_rel is None or src is None or not src.exists():
|
||||
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
|
||||
return
|
||||
staging = _staging_dir(asset_id)
|
||||
placed: Path | None = None # the converted file while it is at its new rel, uncommitted
|
||||
try:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
safe = _ensure_phone_safe(job_id, src, staging, CONVERTING, duration, keep_source=True)
|
||||
ext = safe.suffix.lstrip(".").lower()
|
||||
new_rel = storage.with_ext(old_rel, old_ext, ext)
|
||||
if safe != src:
|
||||
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
|
||||
storage.place_file(safe, settings.files_root, new_rel)
|
||||
if new_rel != old_rel:
|
||||
placed = storage.abs_path(settings.files_root, new_rel)
|
||||
final = storage.abs_path(settings.files_root, new_rel)
|
||||
vcodec, acodec, width, height, dur = probe.probe_media(final)
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "ready"
|
||||
asset.rel_path = new_rel
|
||||
asset.container = ext
|
||||
asset.size_bytes = final.stat().st_size
|
||||
asset.vcodec, asset.acodec = vcodec, acodec
|
||||
asset.width, asset.height = width or asset.width, height or asset.height
|
||||
asset.duration_s = dur or asset.duration_s
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
title = asset.title if asset else None
|
||||
finalized = service.finalize_if_running(
|
||||
db, job_id, status=JobStatus.DONE, progress=100, phase=None
|
||||
)
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
landed = _conversion_landed(asset_id, new_rel)
|
||||
if landed is False:
|
||||
raise # rolled back: the original is still the asset's file — fail as any step
|
||||
# Landed, or cannot tell: the converted file may be the asset's file now, so it is
|
||||
# never removed blind (the sweep takes whichever file no row names).
|
||||
placed = None
|
||||
if landed is None:
|
||||
raise
|
||||
db.rollback()
|
||||
placed = None
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, title, job_id, kind="upload")
|
||||
if new_rel != old_rel:
|
||||
_drop_original(job_id, src)
|
||||
if not finalized:
|
||||
_free_unheld_upload(asset_id) # finished after its job was cancelled or deleted
|
||||
if finalized:
|
||||
log.info(
|
||||
"upload job %s converted: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event("upload.job.done", job_id=job_id, asset_id=asset_id, ext=ext),
|
||||
)
|
||||
except editmod.EditAborted:
|
||||
# Paused/canceled mid-encode: the original is untouched. A paused job keeps it for its
|
||||
# resume; one cancelled or deleted holds it no more, so it is freed now.
|
||||
_unplace(placed)
|
||||
if not _free_unheld_upload(asset_id):
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info(
|
||||
"upload job %s conversion aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("upload.job.aborted", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||
_unplace(placed)
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="upload")
|
||||
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""The phone-safe rule: what a library file must be so it plays wherever a share link lands.
|
||||
|
||||
A shared `/watch` link is opened by whoever got it, on whatever they hold: an iPhone's WebKit, an
|
||||
Android Chrome, a desktop Firefox. The one shape every one of them plays:
|
||||
|
||||
* video → H.264, 8-bit 4:2:0 (`yuv420p`, or its full-range twin `yuvj420p`), AAC audio, in .mp4
|
||||
* audio only → MP3 in .mp3, or AAC in .m4a
|
||||
|
||||
Measured on the owner's iPhone + iPad (2026-10-05): AV1, Matroska, HEVC tagged `hev1` and MP3
|
||||
inside an .mp4 all fail there. VP9/WebM, HEVC `hvc1` and 10-bit H.264 played on those devices, but
|
||||
Chrome and Firefox do not decode HEVC and browsers decode 10-bit / 4:4:4 H.264 only patchily, so the
|
||||
rule is the shape every browser plays, not the shape one device plays.
|
||||
|
||||
Every file that becomes a ready library asset passes through here — a download, an upload (the
|
||||
worker converts it, see `upload`), and an editor clip — so a link can only ever serve this shape.
|
||||
`plan` says what (if anything) must change; `build_cmd` is the one ffmpeg command that changes it:
|
||||
re-encode only the stream that fails, copy the rest, or just remux when only the container is wrong.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import probe
|
||||
|
||||
_SAFE_VCODEC = "h264"
|
||||
_SAFE_PIX_FMTS = frozenset({"yuv420p", "yuvj420p"})
|
||||
_VIDEO_EXT = "mp4"
|
||||
# An audio-only file: the codec, and the extension it plays in. Anything else becomes AAC in .m4a.
|
||||
_AUDIO_HOME = {"mp3": "mp3", "aac": "m4a"}
|
||||
_AUDIO_FALLBACK_EXT = "m4a"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Plan:
|
||||
"""What makes one file phone-safe. `ext` is the extension the safe file has (no dot)."""
|
||||
|
||||
ext: str
|
||||
has_video: bool
|
||||
reencode_video: bool
|
||||
reencode_audio: bool
|
||||
needs_change: bool
|
||||
|
||||
|
||||
def plan(data: dict | None, ext: str) -> Plan | None:
|
||||
"""The phone-safe plan for a file whose `probe.probe_json` is `data` and whose extension is
|
||||
`ext`; None when it is not media at all (an unreadable probe, a still image, no audio or video
|
||||
stream) — such a file is a download-only 'file' and never plays inline, so there is nothing to
|
||||
make safe."""
|
||||
if data is None:
|
||||
return None
|
||||
video, audio = probe.primary_streams(data)
|
||||
ext = ext.lower().lstrip(".")
|
||||
if video is not None:
|
||||
rv = (video.get("codec_name") or "").lower() != _SAFE_VCODEC or (
|
||||
video.get("pix_fmt") or ""
|
||||
).lower() not in _SAFE_PIX_FMTS
|
||||
ra = audio is not None and (audio.get("codec_name") or "").lower() != "aac"
|
||||
return Plan(_VIDEO_EXT, True, rv, ra, rv or ra or ext != _VIDEO_EXT)
|
||||
if audio is not None:
|
||||
home = _AUDIO_HOME.get((audio.get("codec_name") or "").lower())
|
||||
target = home or _AUDIO_FALLBACK_EXT
|
||||
return Plan(target, False, False, home is None, home is None or ext != target)
|
||||
return None
|
||||
|
||||
|
||||
def build_cmd(src: Path, dest: Path, p: Plan, threads: int | None = None) -> list[str]:
|
||||
"""The ffmpeg command that writes `src` to `dest` (whose suffix is `p.ext`) in the phone-safe
|
||||
shape: the failing stream(s) re-encoded with the package's shared encoder settings, the rest
|
||||
stream-copied. An audio-only result drops any cover-art stream. `-progress pipe:1`
|
||||
streams machine-readable progress; `threads` caps the encoder so a re-encode cannot claim every
|
||||
core on the shared box (see the worker-child de-prioritization)."""
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-nostats",
|
||||
*editmod.INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
]
|
||||
# Exactly the two streams `plan` judged: `V` is a video stream that is not cover art. Nothing
|
||||
# else rides along — a subtitle or data track the .mp4 muxer cannot take would fail the run.
|
||||
if p.has_video:
|
||||
cmd += ["-map", "0:V:0", "-map", "0:a:0?"]
|
||||
if p.reencode_video:
|
||||
cmd += ["-vf", editmod.EVEN_FRAME, *editmod.X264_ARGS]
|
||||
else:
|
||||
cmd += ["-c:v", "copy"]
|
||||
else:
|
||||
cmd += ["-map", "0:a:0"]
|
||||
cmd += editmod.AAC_ARGS if p.reencode_audio else ["-c:a", "copy"]
|
||||
if threads and threads > 0:
|
||||
cmd += ["-threads", str(threads)]
|
||||
if p.ext in ("mp4", "m4a"):
|
||||
cmd += ["-movflags", "+faststart"]
|
||||
cmd += [str(dest)]
|
||||
return cmd
|
||||
@@ -0,0 +1,107 @@
|
||||
"""What a library file previews as — the single home of the preview allow-list.
|
||||
|
||||
A generic file (`media_kind == "file"`; video and audio have their players) previews by its stored
|
||||
`container`, the upload's cleaned extension, and ONLY through this table: an image is shown as an
|
||||
image, a PDF in the browser's own viewer, and a text format as plain text. Anything else has no
|
||||
preview. The served Content-Type comes from here too, never from a guess on the name, and HTML / SVG
|
||||
sit in the TEXT list on purpose: rendered on this origin, an uploaded page would run with the
|
||||
reader's session. The route adds `CSP: sandbox` on top, so even a mislabelled file (a "png" that is
|
||||
really a page) gets no script and no origin.
|
||||
|
||||
The serializer hands the kind to the client (`asset.preview`), so the library and the server can
|
||||
never disagree on which files open a preview.
|
||||
"""
|
||||
|
||||
import codecs
|
||||
from typing import Literal
|
||||
|
||||
from app.downloads import mediakind
|
||||
|
||||
PreviewKind = Literal["image", "pdf", "text"]
|
||||
|
||||
IMAGE_TYPES: dict[str, str] = {
|
||||
"png": "image/png",
|
||||
"jpg": "image/jpeg",
|
||||
"jpeg": "image/jpeg",
|
||||
"gif": "image/gif",
|
||||
"webp": "image/webp",
|
||||
"avif": "image/avif",
|
||||
"bmp": "image/bmp",
|
||||
}
|
||||
|
||||
PDF_TYPE = "application/pdf"
|
||||
|
||||
# Read as text: notes, data, subtitles, config and source. HTML and SVG belong here (see above).
|
||||
TEXT_EXTS: frozenset[str] = frozenset(
|
||||
{
|
||||
"txt", "text", "md", "markdown", "log", "csv", "tsv", "json", "jsonl", "ndjson", "xml",
|
||||
"yaml", "yml", "toml", "ini", "conf", "cfg", "properties", "env", "srt", "vtt", "ass", "ssa",
|
||||
"sub", "nfo", "cue", "m3u", "m3u8", "html", "htm", "xhtml", "svg", "css", "scss", "less",
|
||||
"js", "mjs", "cjs", "ts", "tsx", "jsx", "py", "rb", "php", "pl", "lua", "go", "rs", "java",
|
||||
"kt", "kts", "swift", "c", "h", "cc", "cpp", "hpp", "cs", "sh", "bash", "zsh", "fish", "ps1",
|
||||
"bat", "cmd", "sql", "r", "dart", "diff", "patch", "tex", "rst", "adoc", "gradle", "dockerfile",
|
||||
}
|
||||
) # fmt: skip
|
||||
|
||||
TEXT_TYPE = "text/plain; charset=utf-8"
|
||||
|
||||
# A text preview is the file's head: enough to read, never a whole log loaded into a tab.
|
||||
TEXT_PREVIEW_BYTES = 1 << 20
|
||||
# A NUL byte in the first this-many bytes means the "text" is not text.
|
||||
BINARY_SNIFF_BYTES = 8192
|
||||
|
||||
|
||||
def kind(media_kind: str | None, container: str | None) -> PreviewKind | None:
|
||||
"""The preview a library file gets, or None (a media file plays instead; others download)."""
|
||||
if media_kind != mediakind.FILE:
|
||||
return None
|
||||
ext = (container or "").lower()
|
||||
if ext in IMAGE_TYPES:
|
||||
return "image"
|
||||
if ext == "pdf":
|
||||
return "pdf"
|
||||
if ext in TEXT_EXTS:
|
||||
return "text"
|
||||
return None
|
||||
|
||||
|
||||
def media_type(preview: PreviewKind, container: str | None) -> str:
|
||||
"""The Content-Type a preview is served with: from this table, never guessed from the name."""
|
||||
if preview == "image":
|
||||
return IMAGE_TYPES[(container or "").lower()]
|
||||
if preview == "pdf":
|
||||
return PDF_TYPE
|
||||
return TEXT_TYPE
|
||||
|
||||
|
||||
# A text that is not UTF-8 is read as Central European Windows: what an old Hungarian subtitle or
|
||||
# note is written in (its ő and ű are not in Latin-1).
|
||||
FALLBACK_ENCODING = "cp1250"
|
||||
_BOMS = (
|
||||
(codecs.BOM_UTF8, "utf-8-sig"),
|
||||
(codecs.BOM_UTF16_LE, "utf-16"),
|
||||
(codecs.BOM_UTF16_BE, "utf-16"),
|
||||
)
|
||||
|
||||
|
||||
def looks_binary(head: bytes) -> bool:
|
||||
"""Whether the start of a "text" file is binary after all (a NUL in it)."""
|
||||
return b"\x00" in head[:BINARY_SNIFF_BYTES]
|
||||
|
||||
|
||||
def text_utf8(head: bytes) -> bytes | None:
|
||||
"""A text file's head as UTF-8, the one charset the preview is served in; None when it is not text.
|
||||
A byte-order mark names its encoding (UTF-16 is full of NULs, so it is read before the binary
|
||||
check); else UTF-8 is kept as it is — a character cut at the size cap included, the reader shows
|
||||
that one as a replacement mark — and anything else is read as `FALLBACK_ENCODING`."""
|
||||
for bom, encoding in _BOMS:
|
||||
if head.startswith(bom):
|
||||
return head.decode(encoding, errors="replace").encode()
|
||||
if looks_binary(head):
|
||||
return None
|
||||
try:
|
||||
head.decode("utf-8")
|
||||
except UnicodeDecodeError as e: # silent-ok: not UTF-8 is an answer — read it as the fallback
|
||||
if not (e.reason == "unexpected end of data" and e.end == len(head)):
|
||||
return head.decode(FALLBACK_ENCODING, errors="replace").encode()
|
||||
return head
|
||||
@@ -34,6 +34,28 @@ _IMAGE_FORMATS = {
|
||||
}
|
||||
|
||||
|
||||
def primary_streams(data: dict) -> tuple[dict | None, dict | None]:
|
||||
"""(video stream, audio stream) — the first of each kind — from a NON-None ffprobe JSON dict, by
|
||||
the same reading the stored facts use: a still image / image-sequence container has neither (it
|
||||
is a generic 'file'), and a video stream flagged `attached_pic` (an mp3/m4a's cover art) is not a
|
||||
video track. The phone-safe rule (`playable`) judges exactly these two streams."""
|
||||
fmt_names = {
|
||||
n.strip() for n in ((data.get("format") or {}).get("format_name") or "").split(",")
|
||||
}
|
||||
if fmt_names & _IMAGE_FORMATS:
|
||||
return None, None
|
||||
video = audio = None
|
||||
for stream in data.get("streams", []):
|
||||
kind = stream.get("codec_type")
|
||||
if kind == "video" and video is None:
|
||||
if (stream.get("disposition") or {}).get("attached_pic"):
|
||||
continue # cover art, not a video track
|
||||
video = stream
|
||||
elif kind == "audio" and audio is None:
|
||||
audio = stream
|
||||
return video, audio
|
||||
|
||||
|
||||
def _facts_from_probe(
|
||||
data: dict,
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
@@ -45,19 +67,11 @@ def _facts_from_probe(
|
||||
fmt_names = {n.strip() for n in (fmt.get("format_name") or "").split(",")}
|
||||
if fmt_names & _IMAGE_FORMATS:
|
||||
return None, None, None, None, None
|
||||
|
||||
vcodec = acodec = None
|
||||
width = height = None
|
||||
for stream in data.get("streams", []):
|
||||
kind = stream.get("codec_type")
|
||||
if kind == "video" and vcodec is None:
|
||||
if (stream.get("disposition") or {}).get("attached_pic"):
|
||||
continue # cover art, not a video track
|
||||
vcodec = stream.get("codec_name")
|
||||
width = stream.get("width")
|
||||
height = stream.get("height")
|
||||
elif kind == "audio" and acodec is None:
|
||||
acodec = stream.get("codec_name")
|
||||
video, audio = primary_streams(data)
|
||||
vcodec = video.get("codec_name") if video else None
|
||||
width = video.get("width") if video else None
|
||||
height = video.get("height") if video else None
|
||||
acodec = audio.get("codec_name") if audio else None
|
||||
|
||||
duration = None
|
||||
try:
|
||||
@@ -71,16 +85,29 @@ def _facts_from_probe(
|
||||
return vcodec, acodec, width, height, duration
|
||||
|
||||
|
||||
def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
|
||||
or the file isn't playable media (an unprobeable file — or a still image — is treated as a generic
|
||||
non-media file, so it becomes a download-only 'file' rather than a broken 'video')."""
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
def probe_json(path: Path) -> dict | None:
|
||||
"""The full ffprobe JSON (streams + format) of `path`, or None when ffprobe could not read it —
|
||||
the one call both the stored facts (`facts`) and the phone-safe rule read from."""
|
||||
return editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
|
||||
|
||||
def facts(
|
||||
data: dict | None,
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) of a `probe_json` result; all None when it is None
|
||||
(an unprobeable file is treated as a generic non-media file, so it becomes a download-only 'file'
|
||||
rather than a broken 'video')."""
|
||||
if data is None:
|
||||
return None, None, None, None, None
|
||||
return _facts_from_probe(data)
|
||||
|
||||
|
||||
def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
|
||||
or the file isn't playable media (a still image is a generic 'file', see `primary_streams`)."""
|
||||
return facts(probe_json(path))
|
||||
|
||||
|
||||
def resolve_media_facts(
|
||||
path: Path, info: dict
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
@@ -94,18 +121,18 @@ def resolve_media_facts(
|
||||
is authoritative — INCLUDING a deliberate None (cover-art-only audio, a still image). A per-field
|
||||
`probe or info` fallback would let `info`'s stale vcodec resurrect a video track the file doesn't
|
||||
have (yt-dlp reports a progressive `vcodec` for an audio-mode download whose video it then strips),
|
||||
mislabelling an audio download as a broken in-app 'video'. Only when ffprobe could NOT read the
|
||||
file at all do we fall back to `info` (the pre-probe behaviour, a safety net); the cosmetic
|
||||
dimensions and the duration still gap-fill from `info` in either case."""
|
||||
mislabelling an audio download as a broken in-app 'video'. When ffprobe could NOT read the file
|
||||
at all, no codec is reported: the probe runs with the media demuxer whitelist, so a file it
|
||||
cannot open is one the player and the editor cannot open either — and taking `info`'s codecs
|
||||
for it would make it an editable 'video' whose bytes are whatever the remote site served. The
|
||||
cosmetic dimensions and the duration still gap-fill from `info` in either case."""
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
info_v = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||
info_a = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
info_w = int(info["width"]) if info.get("width") else None
|
||||
info_h = int(info["height"]) if info.get("height") else None
|
||||
info_d = int(info["duration"]) if info.get("duration") else None
|
||||
if data is None:
|
||||
# ffprobe could not read the file — fall back to the extractor's info entirely.
|
||||
return info_v, info_a, info_w, info_h, info_d
|
||||
# Unreadable to the guarded probe: a download-only 'file' (no codecs → media_kind FILE).
|
||||
return None, None, info_w, info_h, info_d
|
||||
pv, pa, pw, ph, pd = _facts_from_probe(data)
|
||||
# Probe ran → its codecs win outright (a None means "no such track", not "unknown"); info only
|
||||
# gap-fills the width/height/duration the probe couldn't read.
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Per-user download limits + footprint accounting.
|
||||
|
||||
A user's *footprint* is the total size of the ready assets their library references (cache hits
|
||||
count — it's "how much disk you're responsible for", not "how much you personally downloaded").
|
||||
count — it's "how much disk you're responsible for", not "how much you personally downloaded"). An
|
||||
upload counts from the moment it lands, ready or not: its original is on disk while its conversion
|
||||
waits, and stays there when the conversion fails (for Retry) — bytes a ready-only sum never saw.
|
||||
Limits come from a per-user DownloadQuota row if the admin set one, else the sysconfig defaults.
|
||||
`unlimited` (e.g. the admin's own account) bypasses the byte cap.
|
||||
|
||||
@@ -13,21 +15,30 @@ the footprint as it stood when the job was created and earlier jobs may have fil
|
||||
That is a per-job ceiling, not an aggregate ledger: concurrent jobs of one user each read the same
|
||||
remainder (in-flight bytes are not `ready`, so `footprint` cannot see them), so a wave of
|
||||
max_concurrent downloads can still land over the cap — the deliberate trade, since a download's size
|
||||
is unknown until it finishes. Edit jobs do not consult it at all (tracked as SEC-19).
|
||||
is unknown until it finishes. CLIP jobs consult it before their fetch or encode and again with
|
||||
their output's real size before it is placed. Edit jobs do not consult it at all (tracked as SEC-19).
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.downloads.status import JobKind, JobStatus
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
|
||||
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
||||
# (error, canceled) don't count.
|
||||
_HOLDING = ("queued", "running", "paused", "done")
|
||||
_RUNNING = ("queued", "running")
|
||||
_HOLDING = (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED, JobStatus.DONE)
|
||||
_RUNNING = (JobStatus.QUEUED, JobStatus.RUNNING)
|
||||
# An asset whose file is on disk: a ready one, or an upload in any state (its original lands at
|
||||
# ingest, see the module docstring). Every byte sum — a user's footprint, the GC's total-cap pass,
|
||||
# the admin's storage view — reads it, so they all count the same disk.
|
||||
ON_DISK = or_(
|
||||
MediaAsset.status == "ready",
|
||||
and_(MediaAsset.source_kind == "upload", MediaAsset.rel_path.is_not(None)),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -56,7 +67,8 @@ def resolve(db: Session, user_id: int) -> QuotaLimits:
|
||||
# "at limit" checks — the worker's claim-window exclusion vs at_concurrency_limit — since a
|
||||
# 0-limit user is "capped" with zero running jobs, which the SQL exclusion can't see. Keep
|
||||
# them consistent by never letting the effective limit drop below 1. (To BLOCK a user's
|
||||
# downloads, suspend them or set max_jobs=0, not max_concurrent=0.)
|
||||
# downloads, suspend them — the worker leaves a suspended user's queued jobs unclaimed — or
|
||||
# set max_jobs=0, not max_concurrent=0.)
|
||||
return QuotaLimits(row.max_bytes, max(1, row.max_concurrent), row.max_jobs, row.unlimited)
|
||||
return QuotaLimits(
|
||||
max_bytes=sysconfig.get_int(db, "download_default_max_bytes"),
|
||||
@@ -67,12 +79,17 @@ def resolve(db: Session, user_id: int) -> QuotaLimits:
|
||||
|
||||
|
||||
def footprint(db: Session, user_id: int) -> int:
|
||||
"""Total bytes of the distinct ready assets this user's active jobs reference."""
|
||||
"""Total bytes of the distinct assets this user's jobs hold on disk: the ready ones their active
|
||||
jobs reference, and every upload of theirs — including one whose conversion failed, which keeps
|
||||
its original (see the module docstring)."""
|
||||
held_assets = (
|
||||
select(DownloadJob.asset_id)
|
||||
.where(
|
||||
DownloadJob.user_id == user_id,
|
||||
DownloadJob.status.in_(_HOLDING),
|
||||
or_(
|
||||
DownloadJob.status.in_(_HOLDING),
|
||||
and_(DownloadJob.job_kind == JobKind.UPLOAD, DownloadJob.status == JobStatus.ERROR),
|
||||
),
|
||||
DownloadJob.asset_id.is_not(None),
|
||||
)
|
||||
.distinct()
|
||||
@@ -80,7 +97,7 @@ def footprint(db: Session, user_id: int) -> int:
|
||||
return int(
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.id.in_(held_assets), MediaAsset.status == "ready"
|
||||
MediaAsset.id.in_(held_assets), ON_DISK
|
||||
)
|
||||
).scalar()
|
||||
or 0
|
||||
@@ -111,7 +128,7 @@ def at_concurrency_limit(db: Session, user_id: int) -> bool:
|
||||
lim = resolve(db, user_id)
|
||||
if lim.unlimited:
|
||||
return False
|
||||
running = _count(db, user_id, ("running",))
|
||||
running = _count(db, user_id, (JobStatus.RUNNING,))
|
||||
return running >= lim.max_concurrent
|
||||
|
||||
|
||||
@@ -160,6 +177,18 @@ def check_upload(db: Session, user_id: int, add_bytes: int) -> None:
|
||||
raise QuotaExceeded("max_bytes", lim.max_bytes, projected)
|
||||
|
||||
|
||||
def fits_after_conversion(db: Session, user_id: int, old_bytes: int, new_bytes: int) -> bool:
|
||||
"""Whether an upload converted from `old_bytes` (what it is charged at now) to `new_bytes` keeps
|
||||
its owner within the cap. A re-encode can come out larger than the original, and nothing else
|
||||
re-checks it once the upload was let in."""
|
||||
if new_bytes <= old_bytes: # a smaller file never breaks a cap the larger one was let in under
|
||||
return True
|
||||
lim = resolve(db, user_id)
|
||||
if lim.unlimited:
|
||||
return True
|
||||
return footprint(db, user_id) - old_bytes + new_bytes <= lim.max_bytes
|
||||
|
||||
|
||||
def usage(db: Session, user_id: int) -> dict:
|
||||
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
|
||||
lim = resolve(db, user_id)
|
||||
|
||||
@@ -22,6 +22,7 @@ from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats
|
||||
from app.downloads.status import JobKind
|
||||
from app.models import DownloadJob
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
@@ -57,7 +58,7 @@ def main() -> None:
|
||||
logs.configure_logging("runner")
|
||||
_deprioritize()
|
||||
if len(sys.argv) != 2:
|
||||
# noqa T201: a usage line is for the person who typed the command. Logging is already
|
||||
# T201: a usage line is for the person who typed the command. Logging is already
|
||||
# configured here, and in a deployment it writes JSON records — the one reader of this
|
||||
# line wants the bare sentence, not a record with a request id they do not have.
|
||||
print( # noqa: T201
|
||||
@@ -95,16 +96,20 @@ def _run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id) ->
|
||||
(`cause=child_exit`)."""
|
||||
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
||||
# real work, and to keep the import graph obvious: runner is a thin entry over the pipeline.
|
||||
from app.downloads.pipeline import _download, _process_edit
|
||||
from app.downloads.pipeline import _convert_upload, _download, _process_clip, _process_edit
|
||||
|
||||
with formats.hiding(source_ref):
|
||||
try:
|
||||
if job_kind == "edit":
|
||||
if job_kind == JobKind.EDIT:
|
||||
_process_edit(job_id, asset_id)
|
||||
elif job_kind == JobKind.UPLOAD:
|
||||
_convert_upload(job_id, asset_id)
|
||||
elif job_kind == JobKind.CLIP:
|
||||
_process_clip(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
||||
# noqa TRY400: the stack goes with it scrubbed (`scrubbed_stack`); `exc_info` is raw.
|
||||
# TRY400: the stack goes with it scrubbed (`scrubbed_stack`); `exc_info` is raw.
|
||||
log.error( # noqa: TRY400
|
||||
"the download child for job %s failed",
|
||||
job_id,
|
||||
|
||||
@@ -16,11 +16,22 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import adapters
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobStatus
|
||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||
from app.downloads import folders, formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobKind, JobStatus
|
||||
from app.models import (
|
||||
DISPLAY_TEXT_MAX,
|
||||
Channel,
|
||||
DownloadJob,
|
||||
MediaAsset,
|
||||
PlexItem,
|
||||
PlexShow,
|
||||
Video,
|
||||
)
|
||||
from app.plex import lookups as plex_lookups
|
||||
from app.plex import paths as plex_paths
|
||||
|
||||
|
||||
class EditError(Exception):
|
||||
@@ -38,12 +49,15 @@ def source_url(source_kind: str, source_ref: str) -> str:
|
||||
|
||||
|
||||
def reference_url(job, asset) -> str | None:
|
||||
"""The clean "downloaded from" URL for a job — shared by the private Downloads page and the public
|
||||
"""The clean "downloaded from" URL for a job — shared by the Files library and the public
|
||||
watch page so both surfaces always agree. None for edit derivatives (a clip's source is the user's
|
||||
own earlier download, not a web page); else the canonical page URL yt-dlp recorded; else one derived
|
||||
from source_kind+source_ref (covers queued/older rows before the worker fills it)."""
|
||||
if job is None or job.job_kind == "edit":
|
||||
own earlier download, not a web page) and for a Plex clip (the library is not a public page); a
|
||||
YouTube clip links its video at the clip's start; else the canonical page URL yt-dlp recorded; else
|
||||
one derived from source_kind+source_ref (covers queued/older rows before the worker fills it)."""
|
||||
if job is None or job.job_kind == JobKind.EDIT:
|
||||
return None
|
||||
if job.job_kind == JobKind.CLIP:
|
||||
return clipmod.reference_url(job.edit_spec or {})
|
||||
if asset is not None and asset.source_webpage_url:
|
||||
return asset.source_webpage_url
|
||||
if job.source_kind in ("youtube", "url"):
|
||||
@@ -51,6 +65,23 @@ def reference_url(job, asset) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def library_left_behind(db: Session, files_root: str) -> bool:
|
||||
"""True when the library names files and none of the newest of them is under `files_root` — a
|
||||
fresh, empty volume mounted in the library's place (a compose file changed under an install, its
|
||||
old volume left behind). The mount check cannot see that: the new volume is a real mount."""
|
||||
rels = (
|
||||
db.execute(
|
||||
select(MediaAsset.rel_path)
|
||||
.where(MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None))
|
||||
.order_by(MediaAsset.id.desc())
|
||||
.limit(20)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return bool(rels) and not any(storage.abs_path(files_root, r).exists() for r in rels if r)
|
||||
|
||||
|
||||
def get_or_create_asset(
|
||||
db: Session, source_kind: str, source_ref: str, format_sig: str
|
||||
) -> MediaAsset:
|
||||
@@ -63,7 +94,12 @@ def get_or_create_asset(
|
||||
MediaAsset.source_ref == source_ref,
|
||||
MediaAsset.format_sig == format_sig,
|
||||
)
|
||||
asset = db.execute(stmt).scalar_one_or_none()
|
||||
# Locked to the caller's commit: a release or the GC freeing this asset meanwhile (they lock it
|
||||
# too) either waits for the new hold, or has deleted the row by the time this read returns — and
|
||||
# the caller then gets a fresh one, never a job pointing at a row that is gone.
|
||||
asset = db.execute(
|
||||
stmt.with_for_update().execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if asset is not None:
|
||||
# A previously-FAILED shared asset must be re-attempted for the new job about to hold it —
|
||||
# reset it to pending (+clear the error) so the worker actually re-downloads instead of
|
||||
@@ -179,17 +215,19 @@ def enqueue(
|
||||
profile_id=profile_id,
|
||||
profile_snapshot=spec,
|
||||
display_name=display_name
|
||||
or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
||||
status="queued",
|
||||
or (asset.title[:DISPLAY_TEXT_MAX] if asset.status == "ready" and asset.title else None),
|
||||
status=JobStatus.QUEUED,
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
folders.place(db, job)
|
||||
incr_ref_count(db, asset.id)
|
||||
|
||||
# Cache hit: the file already exists — no worker round-trip needed.
|
||||
if asset.status == "ready":
|
||||
job.status = "done"
|
||||
job.status = JobStatus.DONE
|
||||
job.progress = 100
|
||||
folders.stamp_done(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
@@ -197,11 +235,14 @@ def enqueue(
|
||||
return job
|
||||
|
||||
|
||||
def _clip_title(source_title: str | None, display_name: str | None) -> str:
|
||||
def clip_title(source_title: str | None, display_name: str | None) -> str:
|
||||
# Clipped to DISPLAY_TEXT_MAX, not to a literal: this feeds `DownloadJob.display_name` through
|
||||
# `enqueue_edit`, the same column `display_text` guards. A machine-made title, so it is clipped
|
||||
# rather than run through `display_text` — blank here means "no title", not None.
|
||||
if display_name:
|
||||
return display_name[:255]
|
||||
return display_name[:DISPLAY_TEXT_MAX]
|
||||
base = source_title or "Clip"
|
||||
return f"{base} (clip)"[:255]
|
||||
return f"{base} (clip)"[:DISPLAY_TEXT_MAX]
|
||||
|
||||
|
||||
def enqueue_edit(
|
||||
@@ -236,7 +277,7 @@ def enqueue_edit(
|
||||
ref = f"{user_id}:{src_asset.id}"[:512]
|
||||
asset = get_or_create_asset(db, "edit", ref, sig)
|
||||
if not asset.title:
|
||||
asset.title = _clip_title(src_asset.title, display_name)
|
||||
asset.title = clip_title(src_asset.title, display_name)
|
||||
asset.uploader = src_asset.uploader
|
||||
asset.thumbnail_url = src_asset.thumbnail_url
|
||||
asset.upload_date = src_asset.upload_date
|
||||
@@ -248,23 +289,114 @@ def enqueue_edit(
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind="edit",
|
||||
job_kind=JobKind.EDIT,
|
||||
source_kind="edit",
|
||||
source_ref=ref,
|
||||
source_job_id=source_job.id,
|
||||
source_asset_id=src_asset.id,
|
||||
edit_spec=spec,
|
||||
profile_snapshot={},
|
||||
display_name=display_name or asset.title,
|
||||
status="queued",
|
||||
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
|
||||
status=JobStatus.QUEUED,
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
folders.place(db, job)
|
||||
incr_ref_count(db, asset.id)
|
||||
|
||||
if asset.status == "ready": # identical clip already produced for this user → instant
|
||||
job.status = "done"
|
||||
job.status = JobStatus.DONE
|
||||
job.progress = 100
|
||||
folders.stamp_done(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
def _clip_source_facts(db: Session, spec: dict) -> dict:
|
||||
"""Title, uploader, thumbnail and length of what a clip is cut from, read from our own catalog.
|
||||
Raises ClipError when a Plex item is unknown or its file cannot be read here — said at enqueue,
|
||||
rather than as a job that fails in the queue a minute later."""
|
||||
if spec["source"] == "youtube":
|
||||
video = db.get(Video, spec["ref"])
|
||||
if video is None: # not in the catalog: titled "Clip" until yt-dlp names it (worker)
|
||||
return {"title": None, "uploader": None, "thumbnail_url": None, "length_s": None}
|
||||
channel = db.get(Channel, video.channel_id)
|
||||
return {
|
||||
"title": video.title,
|
||||
"uploader": channel.title if channel else None,
|
||||
"thumbnail_url": video.thumbnail_url,
|
||||
"length_s": video.duration_seconds,
|
||||
}
|
||||
item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"])
|
||||
if item is None:
|
||||
raise clipmod.ClipError("not_found")
|
||||
if plex_paths.local_media_path(db, item.file_path) is None:
|
||||
raise clipmod.ClipError("media_missing")
|
||||
title = item.title
|
||||
show = db.get(PlexShow, item.show_id) if item.show_id else None
|
||||
if show is not None:
|
||||
episode = (
|
||||
f"S{item.season_number}·E{item.episode_number}"
|
||||
if item.season_number is not None and item.episode_number is not None
|
||||
else None
|
||||
)
|
||||
title = " — ".join(p for p in (show.title, episode, item.title) if p)
|
||||
return {
|
||||
"title": title,
|
||||
"uploader": None,
|
||||
# Plex art is served through our own authenticated routes, never a public URL; the worker
|
||||
# cuts the clip's poster from the clip itself.
|
||||
"thumbnail_url": None,
|
||||
"length_s": item.duration_s,
|
||||
}
|
||||
|
||||
|
||||
def enqueue_clip(
|
||||
db: Session, user_id: int, spec: dict, display_name: str | None = None
|
||||
) -> DownloadJob:
|
||||
"""Queue an ad-hoc clip (`clipmod.normalize`d spec) for `user_id`: a NEW per-user asset cut
|
||||
straight from the source — a YouTube range or a Plex library file (see downloads/clip).
|
||||
|
||||
Raises ClipError (unknown or unreachable source, a range past its end) or quota.QuotaExceeded."""
|
||||
facts = _clip_source_facts(db, spec)
|
||||
# One second of grace: a mark set on the last frame may round past the catalog's whole seconds.
|
||||
if facts["length_s"] and spec["end_s"] > facts["length_s"] + 1:
|
||||
raise clipmod.ClipError("past_end")
|
||||
|
||||
quota.check_enqueue(db, user_id)
|
||||
|
||||
ref = clipmod.asset_ref(user_id, spec)[:512]
|
||||
asset = get_or_create_asset(db, "clip", ref, clipmod.sig(spec))
|
||||
if not asset.title:
|
||||
asset.title = clip_title(facts["title"], display_name)
|
||||
asset.uploader = facts["uploader"]
|
||||
asset.thumbnail_url = facts["thumbnail_url"]
|
||||
asset.duration_s = round(clipmod.duration(spec))
|
||||
asset.media_kind = mediakind.VIDEO
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind=JobKind.CLIP,
|
||||
source_kind="clip",
|
||||
source_ref=ref,
|
||||
edit_spec=spec,
|
||||
profile_snapshot={},
|
||||
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
|
||||
status=JobStatus.QUEUED,
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
folders.place(db, job)
|
||||
incr_ref_count(db, asset.id)
|
||||
|
||||
if asset.status == "ready": # the same range was already cut for this user → instant
|
||||
job.status = JobStatus.DONE
|
||||
job.progress = 100
|
||||
folders.stamp_done(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
@@ -288,6 +420,8 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
|
||||
if not fields:
|
||||
return False
|
||||
db.flush() # asset-then-job lock order: push pending asset writes before we lock the job row
|
||||
if fields.get("status") == JobStatus.DONE:
|
||||
return _finalize_done(db, job_id, {**fields, "ready_at": datetime.now(timezone.utc)})
|
||||
# A Core UPDATE with the guard as a WHERE — same atomic "only while still running" write as the
|
||||
# rest of the codebase's 2.0 statements, no hand-built SQL string. synchronize_session=False: this
|
||||
# is a terminal write, so we don't pay to sync the (about-to-be-discarded) ORM identity map. Bound
|
||||
@@ -301,6 +435,41 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
|
||||
return result.rowcount > 0
|
||||
|
||||
|
||||
def _finalize_done(db: Session, job_id: int, fields: dict) -> bool:
|
||||
"""`finalize_if_running` for a job that finished: done is when a file's retention starts, its
|
||||
folder's unless the user kept it. The expiry is written only while the job is still where it was
|
||||
read from — a move, a keep, or its folder's delete (which sends its files home) landing while the
|
||||
folder's retention was read sends the read round again, never stamping a folder's retention on a
|
||||
file now in another one."""
|
||||
while True:
|
||||
held = db.execute(
|
||||
select(DownloadJob.folder_id, DownloadJob.kept, DownloadJob.status).where(
|
||||
DownloadJob.id == job_id
|
||||
)
|
||||
).one_or_none()
|
||||
if held is None or held.status != JobStatus.RUNNING:
|
||||
return False
|
||||
stamped = {
|
||||
**fields,
|
||||
"expires_at": folders.done_expiry(db, held.folder_id, held.kept),
|
||||
"expiry_warned": False,
|
||||
}
|
||||
hit = db.execute(
|
||||
update(DownloadJob)
|
||||
.where(
|
||||
DownloadJob.id == job_id,
|
||||
DownloadJob.status == JobStatus.RUNNING.value,
|
||||
DownloadJob.folder_id.is_not_distinct_from(held.folder_id),
|
||||
DownloadJob.kept == held.kept,
|
||||
)
|
||||
.values(**stamped)
|
||||
.returning(DownloadJob.id)
|
||||
.execution_options(synchronize_session=False)
|
||||
).scalar_one_or_none()
|
||||
if hit is not None:
|
||||
return True
|
||||
|
||||
|
||||
def release_asset(db: Session, job: DownloadJob) -> None:
|
||||
"""Drop this job's hold on its asset when it leaves a holding state. Once NO job holds the
|
||||
asset anymore, delete the file + row immediately — a deleted download should free its disk,
|
||||
@@ -315,16 +484,24 @@ def release_asset(db: Session, job: DownloadJob) -> None:
|
||||
re-enqueuing the same (source,format) — get_or_create_asset would reset that row to `pending` and
|
||||
+1 it, and deleting it under that would FK-null the new job's asset (a lost update on ref_count).
|
||||
A ready asset is still freed at ref==0 (its file is the point); an orphaned errored row is cheap
|
||||
(no file) and gets reused+reset by the next enqueue, or reclaimed by a later GC pass."""
|
||||
(no file) and gets reused+reset by the next enqueue, or reclaimed by a later GC pass.
|
||||
|
||||
An UPLOAD is the exception: it is never shared or re-enqueued (its ref is unique), and while it
|
||||
waits for or failed its phone-safe conversion the original file already sits at `rel_path` — so
|
||||
it is freed at ref==0 whatever its state, except mid-conversion (`downloading`: the worker child
|
||||
still reads it; the GC upload pass takes it once the child has stopped)."""
|
||||
if not job.asset_id or job.status not in HOLDING:
|
||||
return
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
if asset is None:
|
||||
return
|
||||
new_count = decr_ref_count(db, asset.id)
|
||||
if new_count == 0 and asset.status == "ready":
|
||||
freeable = asset.status == "ready" or (
|
||||
asset.source_kind == "upload" and asset.status != "downloading"
|
||||
)
|
||||
if new_count == 0 and freeable:
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
|
||||
|
||||
@@ -453,4 +630,6 @@ def redownload(db: Session, job: DownloadJob) -> None:
|
||||
# name would survive the rebuild).
|
||||
job.display_name = None
|
||||
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
|
||||
job.expires_at = None # stamped again when the rebuild is done
|
||||
job.expiry_warned = False
|
||||
db.commit()
|
||||
|
||||
@@ -33,6 +33,27 @@ HOLDING = frozenset(
|
||||
{JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED, JobStatus.DONE, JobStatus.ERROR}
|
||||
)
|
||||
|
||||
|
||||
class JobKind(StrEnum):
|
||||
"""What a job produces, which decides the worker path that runs it and what the library may do
|
||||
with the result (only a DOWNLOAD has a web source to fetch again)."""
|
||||
|
||||
DOWNLOAD = "download" # yt-dlp fetches a source into a shared, deduplicated asset
|
||||
EDIT = "edit" # a per-user trim/crop of a file already in the library
|
||||
UPLOAD = "upload" # a file the user sent; the worker only converts it
|
||||
CLIP = (
|
||||
"clip" # a per-user range of a YouTube video or a Plex item, cut straight from the source
|
||||
)
|
||||
|
||||
|
||||
# A job kind with no web source to fetch again: Re-download refuses these.
|
||||
NOT_REFETCHABLE = frozenset({JobKind.EDIT, JobKind.UPLOAD, JobKind.CLIP})
|
||||
|
||||
|
||||
# States a route sets to STOP a running job. The worker's supervisor must step in (kill the child, free
|
||||
# the asset, leave the route's status intact) when it observes one of these on a job it is running.
|
||||
STOPPED_BY_ROUTE = frozenset({JobStatus.PAUSED, JobStatus.CANCELED})
|
||||
|
||||
|
||||
# A job that will not move again on its own: the queue lists it only briefly after it got here.
|
||||
FINISHED = frozenset({JobStatus.DONE, JobStatus.CANCELED})
|
||||
|
||||
@@ -11,6 +11,7 @@ a Plex library using a metadata agent picks up rich info. Channel = "show", vide
|
||||
date-based — the community-standard mapping for YouTube-in-Plex.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import unicodedata
|
||||
@@ -67,6 +68,57 @@ _FOLD_MAP = str.maketrans(
|
||||
)
|
||||
|
||||
|
||||
def _mount_points(mountinfo: str) -> list[str]:
|
||||
"""The mount points a /proc/<pid>/mountinfo table lists (its fifth field, octal escapes undone)."""
|
||||
points = []
|
||||
for line in mountinfo.splitlines():
|
||||
fields = line.split(" ")
|
||||
if len(fields) > 4:
|
||||
points.append(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), fields[4]))
|
||||
return points
|
||||
|
||||
|
||||
def _in_container() -> bool:
|
||||
return Path("/.dockerenv").exists() or Path("/run/.containerenv").exists()
|
||||
|
||||
|
||||
def _read_mountinfo() -> str | None:
|
||||
try:
|
||||
return Path("/proc/self/mountinfo").read_text()
|
||||
# silent-ok: no mount table to read (not Linux) — the caller then cannot tell, and refuses nothing.
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def files_root_problem(
|
||||
root: str, mountinfo: str | None = None, in_container: bool | None = None
|
||||
) -> str | None:
|
||||
"""Why `root` cannot be the files library's volume, or None when it can. In a container, a root
|
||||
that is no mount of its own (nor inside one) is the image's empty directory: the library would
|
||||
answer 404 for every file, and what a download writes would go when the container does — the
|
||||
state an install is in when its compose file still mounts the library somewhere else.
|
||||
|
||||
Asked of the kernel's mount table, not by comparing devices: a storage driver that keeps the
|
||||
container's root on the host's own disk (vfs, the fallback under Docker-in-LXC on older ZFS) gives
|
||||
a mounted volume the same device as `/`. Outside a container a plain directory is a fine root."""
|
||||
path = Path(root)
|
||||
if not path.is_dir():
|
||||
return f"{root} does not exist"
|
||||
if in_container if in_container is not None else _in_container():
|
||||
table = mountinfo if mountinfo is not None else _read_mountinfo()
|
||||
if table is not None:
|
||||
real = os.path.realpath(root)
|
||||
mounted = any(
|
||||
m != "/" and (real == m or real.startswith(m.rstrip("/") + "/"))
|
||||
for m in _mount_points(table)
|
||||
)
|
||||
if not mounted:
|
||||
return f"{root} is not a mounted volume (it is the container's own empty directory)"
|
||||
if not os.access(path, os.W_OK):
|
||||
return f"{root} is not writable"
|
||||
return None
|
||||
|
||||
|
||||
def _ascii_fold(text: str) -> str:
|
||||
"""Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
|
||||
the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have
|
||||
@@ -122,7 +174,7 @@ def display_filename(name: str) -> str:
|
||||
|
||||
|
||||
def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | None = None) -> str:
|
||||
"""Path of the media file relative to DOWNLOAD_ROOT (forward slashes).
|
||||
"""Path of the media file relative to FILES_ROOT (forward slashes).
|
||||
|
||||
`asset_id` disambiguates the file. A MediaAsset is unique on (source_kind, source_ref,
|
||||
format_sig), so the SAME video downloaded in two formats is two distinct assets — but without
|
||||
@@ -144,7 +196,7 @@ def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | No
|
||||
|
||||
|
||||
def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
|
||||
"""On-disk path (relative to DOWNLOAD_ROOT) for a phase-2 editor derivative.
|
||||
"""On-disk path (relative to FILES_ROOT) for a phase-2 editor derivative.
|
||||
|
||||
Kept in a per-user `.edits/` tree — device-downloadable via the file endpoint, but NOT part of
|
||||
the Plex library (these are user clips, not canonical episodes; there are no .nfo/poster
|
||||
@@ -161,20 +213,32 @@ def _clean_upload_ext(filename: str) -> str:
|
||||
|
||||
|
||||
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
|
||||
"""On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
|
||||
"""On-disk path (relative to FILES_ROOT) for a locally-uploaded file.
|
||||
|
||||
Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of
|
||||
the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster
|
||||
episode sidecars). The name is system-decided and bound to the asset id (`_a{id}` keeps two
|
||||
same-named uploads apart); the original filename is display metadata only. The extension is
|
||||
preserved (shell-safe, bounded) so the served file + its Content-Disposition carry a correct
|
||||
suffix; an extensionless upload stays extensionless."""
|
||||
stem = sanitize(Path(filename).stem, 80)
|
||||
suffix; an extensionless upload stays extensionless.
|
||||
|
||||
The stem holds no dot, so the last dot of the name is always the extension's: every sibling
|
||||
derived by swapping it (`with_suffix` — the poster, the converted file) keeps `_a{id}`. A dotted
|
||||
stem with no extension ('Holiday.2024.ä') once lost it, and two such uploads shared one file."""
|
||||
stem = sanitize(Path(filename).stem, 80).replace(".", "_")
|
||||
ext = _clean_upload_ext(filename)
|
||||
suffix = f".{ext}" if ext else ""
|
||||
return f".uploads/{user_id}/{stem}_a{asset_id}{suffix}"
|
||||
|
||||
|
||||
def with_ext(rel: str, old_ext: str | None, ext: str) -> str:
|
||||
"""`rel` with its extension — `old_ext`, the stored `container`, None for a bare name — swapped
|
||||
for `ext`. Not `Path.with_suffix`: on a bare name whose stem holds a dot (an upload placed before
|
||||
the stems lost their dots) that would take the stem's tail for the extension, and `_a{id}` with it."""
|
||||
base = rel[: -len(old_ext) - 1] if old_ext and rel.endswith(f".{old_ext}") else rel
|
||||
return f"{base}.{ext}"
|
||||
|
||||
|
||||
def upload_ext(filename: str) -> str | None:
|
||||
"""The stored `container` for an uploaded file: its shell-safe lowercased extension, or None when
|
||||
the upload has no extension (a bare name). Shares `_clean_upload_ext` with `upload_rel_path` so the
|
||||
@@ -230,32 +294,39 @@ def served_media_type(media_kind: str | None, container: str | None) -> str | No
|
||||
def download_filename(display_name: str, container: str | None, path: Path) -> str:
|
||||
"""The Content-Disposition filename for a served download: the cleaned display name carrying a single
|
||||
correct extension — the asset container, else the file's own suffix — without doubling it when the
|
||||
name already ends in that extension. Shared by the private download and the public watch endpoints."""
|
||||
name already ends in that extension. A file with no extension at all is served without one —
|
||||
never as `name.` (Windows drops a trailing dot, so the saved file would not be the one named).
|
||||
Shared by the private download and the public watch endpoints."""
|
||||
ext = container or path.suffix.lstrip(".")
|
||||
base = display_filename(display_name)
|
||||
if not ext:
|
||||
return base
|
||||
if base.lower().endswith(f".{ext.lower()}"):
|
||||
base = base[: -(len(ext) + 1)]
|
||||
return f"{base}.{ext}"
|
||||
|
||||
|
||||
def abs_path(download_root: str, rel: str) -> Path:
|
||||
return Path(download_root) / rel
|
||||
def abs_path(files_root: str, rel: str) -> Path:
|
||||
return Path(files_root) / rel
|
||||
|
||||
|
||||
def safe_abs_path(download_root: str, rel: str) -> Path | None:
|
||||
"""The resolved absolute path for `rel`, but ONLY if it stays inside DOWNLOAD_ROOT and exists on
|
||||
disk — otherwise None. Centralizes the path-traversal + existence guard every file-serving endpoint
|
||||
shares, so the containment check lives in exactly one place."""
|
||||
root = Path(download_root).resolve()
|
||||
path = abs_path(download_root, rel).resolve()
|
||||
def safe_abs_path(files_root: str, rel: str | None) -> Path | None:
|
||||
"""The resolved absolute path for `rel`, but ONLY if it stays inside FILES_ROOT and exists on
|
||||
disk — otherwise None (no `rel` at all included: an asset without a file has nothing to serve).
|
||||
Centralizes the path-traversal + existence guard every file-serving endpoint shares, so the
|
||||
containment check lives in exactly one place."""
|
||||
if not rel:
|
||||
return None
|
||||
root = Path(files_root).resolve()
|
||||
path = abs_path(files_root, rel).resolve()
|
||||
if root not in path.parents or not path.exists():
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def place_file(staging_file: Path, download_root: str, rel: str) -> None:
|
||||
"""Move a completed staging file into its final location under DOWNLOAD_ROOT."""
|
||||
dest = abs_path(download_root, rel)
|
||||
def place_file(staging_file: Path, files_root: str, rel: str) -> None:
|
||||
"""Move a completed staging file into its final location under FILES_ROOT."""
|
||||
dest = abs_path(files_root, rel)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(staging_file), str(dest))
|
||||
|
||||
@@ -289,9 +360,9 @@ def _sidecar(media: Path, ext: str) -> Path:
|
||||
return media.with_suffix(ext)
|
||||
|
||||
|
||||
def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
|
||||
def write_sidecars(files_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
|
||||
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
||||
media = abs_path(download_root, rel)
|
||||
media = abs_path(files_root, rel)
|
||||
try:
|
||||
_sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
|
||||
if thumb_src and thumb_src.exists():
|
||||
@@ -307,9 +378,9 @@ def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Pat
|
||||
return False
|
||||
|
||||
|
||||
def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
def delete_asset_files(files_root: str, rel: str) -> None:
|
||||
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
|
||||
media = abs_path(download_root, rel)
|
||||
media = abs_path(files_root, rel)
|
||||
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
|
||||
try:
|
||||
p.unlink()
|
||||
@@ -323,7 +394,7 @@ def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__),
|
||||
)
|
||||
# Prune empty Season/Channel dirs up to (but not including) the root.
|
||||
root = Path(download_root).resolve()
|
||||
root = Path(files_root).resolve()
|
||||
parent = media.parent
|
||||
while parent != root and root in parent.parents:
|
||||
try:
|
||||
|
||||
@@ -1,15 +1,24 @@
|
||||
"""Local file ingest for the download center (E7 S5) — the non-yt-dlp path into the media library.
|
||||
"""Local file ingest for the download center — the non-yt-dlp path into the media library.
|
||||
|
||||
A user-uploaded file joins the SAME library as a downloaded one, so it inherits the whole function
|
||||
set already built (metadata chips, share links, and — for media — in-app play/edit). Unlike a
|
||||
download it bypasses yt-dlp AND the worker entirely: the file is already on the host, so ingest
|
||||
probes it, places it on disk, and creates an already-`ready` MediaAsset + `done` DownloadJob in one
|
||||
step — the same "the file exists, mark it done now" shortcut `service.enqueue` takes on a cache hit.
|
||||
download it bypasses yt-dlp: the file is already on the host, so ingest probes it and places it on
|
||||
disk. A file that already is the phone-safe shape (`playable`) — or is not media at all — becomes an
|
||||
already-`ready` MediaAsset + `done` DownloadJob in one step, the same "the file exists, mark it done
|
||||
now" shortcut `service.enqueue` takes on a cache hit.
|
||||
|
||||
Media that is NOT phone-safe (an iPhone's HEVC .mov, a VP9 .webm, an .mkv) is placed as it came and
|
||||
queued: the asset stays `pending` and the job `queued`, and the download worker converts it
|
||||
(`pipeline._convert_upload`) — so a share link never hands a friend a file their phone cannot play.
|
||||
Only the converted file is kept (the owner's choice: one file per upload). A 10-bit HDR original
|
||||
becomes 8-bit H.264 with its colour tags kept — still tagged HDR (HLG/PQ, BT.2020), not tone-mapped
|
||||
to SDR (the owner's choice too). Until then the original sits at the asset's `rel_path`, which is
|
||||
what a failed conversion's Retry starts from again, and what a delete removes
|
||||
(`service.release_asset`).
|
||||
|
||||
Kinds (the invariant lives in `mediakind`): a probe with a real video stream -> video, audio-only
|
||||
-> audio, neither -> a generic 'file' (pdf/zip/docx/…) that is download-only and never rendered
|
||||
inline (see the force-download serving in the file/watch routes). This module is the ingest seam a
|
||||
future sub-function (e.g. convert) plugs a new adapter into, mirroring how it sidesteps the worker.
|
||||
inline (see the force-download serving in the file/watch routes).
|
||||
|
||||
An upload does NOT get an `expires_at`: it's user-provided and may be irreplaceable (there's no
|
||||
source to re-download), so it doesn't age out on the retention clock the way a re-fetchable download
|
||||
@@ -22,10 +31,13 @@ from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit
|
||||
from app.audit import AuditAction
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, probe, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
|
||||
from app.downloads.status import JobKind, JobStatus
|
||||
from app.models import DownloadJob, MediaAsset, display_text
|
||||
|
||||
|
||||
def _title_from_filename(filename: str) -> str:
|
||||
@@ -43,16 +55,35 @@ def ingest_upload(
|
||||
src_path: Path,
|
||||
size: int,
|
||||
display_name: str | None = None,
|
||||
sha256: str | None = None,
|
||||
folder_id: int | None = None,
|
||||
via: str | None = None,
|
||||
) -> DownloadJob:
|
||||
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return the `done`
|
||||
job. `src_path` is MOVED into place on success (the caller stages it to a temp file first).
|
||||
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return its job —
|
||||
`done`, or `queued` for the worker's phone-safe conversion (see the module docstring). `src_path`
|
||||
is MOVED into place on success (the caller stages it to a temp file first). `sha256` is the
|
||||
hash of those bytes, when the caller took it on the way in; `folder_id` the folder it asked for
|
||||
(`folders.upload_target`), else its Uploads folder; `via` what the audit row names as the doer
|
||||
when it was not the user themselves (an API token).
|
||||
|
||||
Raises folders.FolderError when that folder is not one an upload may land in (or is gone).
|
||||
|
||||
Raises quota.QuotaExceeded (holdings/footprint cap) — checked BEFORE any disk write, so a
|
||||
rejected upload leaves nothing behind and the caller just cleans up its temp file."""
|
||||
quota.check_upload(db, user_id, size) # the file's size is known → charge it up front
|
||||
if folder_id is not None:
|
||||
# Refused before any work; held only once the job is about to name it (below), not through
|
||||
# the probe and the poster, which a rename or delete of the folder would wait out.
|
||||
folders.upload_target(db, user_id, folder_id, hold=False)
|
||||
|
||||
vcodec, acodec, width, height, duration = probe.probe_media(src_path)
|
||||
data = probe.probe_json(src_path)
|
||||
vcodec, acodec, width, height, duration = probe.facts(data)
|
||||
kind = mediakind.derive(vcodec, acodec)
|
||||
# Judged by the extension the file is PLACED under (and the worker re-judges), not the raw one:
|
||||
# 'clip.mp4~' is stored as .mp4, and judging 'mp4~' queued a conversion with nothing to convert.
|
||||
ext = storage.upload_ext(filename) or ""
|
||||
safe = playable.plan(data, ext) if kind != mediakind.FILE else None
|
||||
convert = safe is not None and safe.needs_change
|
||||
|
||||
# An upload never dedups (each file is its own asset): a random ref makes the cache key unique,
|
||||
# so `format_sig` is a constant marker rather than a format hash.
|
||||
@@ -64,19 +95,23 @@ def ingest_upload(
|
||||
db.flush() # need asset.id for the on-disk path
|
||||
|
||||
rel = storage.upload_rel_path(user_id, asset.id, filename)
|
||||
storage.place_file(src_path, settings.download_root, rel)
|
||||
storage.place_file(src_path, settings.files_root, rel)
|
||||
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
|
||||
# asset/job rows roll back but the placed file would be ORPHANED — nothing references it, the
|
||||
# per-user delete path keys off asset.rel_path (which is gone), and the GC orphan sweep skips the
|
||||
# dot-prefixed `.uploads/` tree. So roll back the on-disk side too on any failure.
|
||||
# asset/job rows roll back and nothing names the placed file: the per-user delete path keys off
|
||||
# asset.rel_path (which is gone), and the GC's `.uploads` sweep reaches it only an hour later. So
|
||||
# roll back the on-disk side too — unless the commit itself raised after it landed (see below).
|
||||
asset_id = asset.id
|
||||
try:
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
actual_size = final.stat().st_size if final.exists() else size
|
||||
|
||||
asset.status = "ready"
|
||||
asset.status = "pending" if convert else "ready"
|
||||
asset.rel_path = rel
|
||||
# The hash names the bytes that arrived: a conversion replaces them, so it is kept only
|
||||
# when the file stays as it came.
|
||||
asset.sha256 = None if convert else sha256
|
||||
asset.size_bytes = actual_size
|
||||
asset.container = storage.upload_ext(filename)
|
||||
asset.container = ext or None
|
||||
asset.vcodec = vcodec
|
||||
asset.acodec = acodec
|
||||
asset.width = width
|
||||
@@ -90,25 +125,44 @@ def ingest_upload(
|
||||
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
|
||||
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
|
||||
# server side even for a codec the browser can't play).
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind="upload",
|
||||
job_kind=JobKind.UPLOAD,
|
||||
source_kind="upload",
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
display_name=(display_name or "").strip() or None,
|
||||
status="done",
|
||||
progress=100,
|
||||
display_name=display_text(display_name),
|
||||
status=JobStatus.QUEUED if convert else JobStatus.DONE,
|
||||
progress=0 if convert else 100,
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
folders.place(db, job)
|
||||
if folder_id is not None:
|
||||
job.folder_id = folders.upload_target(db, user_id, folder_id).id
|
||||
if not convert:
|
||||
folders.stamp_done(db, job)
|
||||
service.incr_ref_count(db, asset.id)
|
||||
db.flush()
|
||||
audit.record(
|
||||
db,
|
||||
user_id,
|
||||
AuditAction.FILE_UPLOAD,
|
||||
target_type="file",
|
||||
target_id=job.id,
|
||||
summary=via,
|
||||
)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
if _row_landed(db, asset_id) is not False:
|
||||
# The commit reached the server and lost its answer, or the row cannot be read: the file
|
||||
# may be the asset's now, so it is never removed blind — the sweep takes it if no row
|
||||
# names it.
|
||||
raise
|
||||
# The rows are gone; remove the now-unreferenced file so no bytes leak. Clean the poster by its
|
||||
# DETERMINISTIC path (rel with a .jpg suffix, the same path ensure_poster writes to) rather than
|
||||
# ensure_poster's return value: it writes the frame directly to that path and returns None on an
|
||||
@@ -116,7 +170,17 @@ def ingest_upload(
|
||||
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
|
||||
# when none was generated (audio/file, or a video whose poster never started).
|
||||
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
|
||||
storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True)
|
||||
storage.abs_path(settings.files_root, leaked).unlink(missing_ok=True)
|
||||
raise
|
||||
db.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
def _row_landed(db: Session, asset_id: int) -> bool | None:
|
||||
"""After the ingest transaction raised: is the asset row there anyway? A commit can reach the
|
||||
server and lose its answer (the connection drops after COMMIT). None when it cannot be read."""
|
||||
try:
|
||||
return db.get(MediaAsset, asset_id) is not None
|
||||
except Exception: # noqa: BLE001 — "cannot tell" is the answer, the caller acts on it
|
||||
# silent-ok: the caller re-raises the ingest's own error.
|
||||
return None
|
||||
|
||||
@@ -197,7 +197,7 @@ def _next_step(verify_link: str | None, has_account: bool, hours: int, lang: str
|
||||
|
||||
Three states, because a mail that guesses wrong sends someone looking for a link that isn't
|
||||
coming — or worse, tells them to sign in when their address is still unconfirmed and sign-in will
|
||||
refuse them (E9.5 S4 named that state on screen; the mail must not contradict it).
|
||||
refuse them (the sign-in screen names that state; the mail must not contradict it).
|
||||
|
||||
The link branch is the point of the C option: in the common ordering — approve first, the person
|
||||
signs up later, or signs up and waits — they would otherwise have to find and open TWO separate
|
||||
@@ -253,7 +253,7 @@ def send_access_invitation(
|
||||
the closing line offers no deadline — nudging is the one thing this mail must not do.
|
||||
|
||||
Its language is the one place the app has no signal to read: the admin typed this address in, so
|
||||
nobody has ever told us how to address its owner. That is why the send button asks (E9.5 S7).
|
||||
nobody has ever told us how to address its owner. That is why the send button asks.
|
||||
"""
|
||||
admin, contact = _reply_contact(lang)
|
||||
body = mailtext.text(
|
||||
|
||||
+107
-23
@@ -12,9 +12,10 @@ import binascii
|
||||
import json
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
|
||||
from sqlalchemy import BigInteger, Select, and_, case, cast, false, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.collation import collated
|
||||
from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
@@ -105,15 +106,30 @@ def _tag_visible(user: User):
|
||||
return or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||
|
||||
|
||||
def tags_by_category(db: Session, tag_ids: list[int]) -> dict[str, list[int]]:
|
||||
# The pseudo-category of a selected tag id no Tag row answers (a saved view or the feed state outlives
|
||||
# a deleted tag). It filters like a category whose tags no channel carries — to nothing, in the SQL and
|
||||
# in feed_summary's mirror alike — instead of the id being dropped, which showed the WHOLE feed under
|
||||
# the view's old name. No real category is called this (language/topic/other).
|
||||
MISSING_TAG_CATEGORY = "missing"
|
||||
|
||||
|
||||
def tags_by_category(db: Session, user: User, tag_ids: list[int]) -> dict[str, list[int]]:
|
||||
"""Group tag ids by their category (language/topic/other). The single source for the category split
|
||||
shared by the feed's tag WHERE (filtered_query) and the facet/count summary (feed_summary), so the
|
||||
two can't disagree on which selected tag belongs to which category."""
|
||||
two can't disagree on which selected tag belongs to which category. An unknown id lands in
|
||||
MISSING_TAG_CATEGORY — and so does another user's private tag: resolving it to its real category
|
||||
told the caller which foreign ids exist."""
|
||||
if not tag_ids:
|
||||
return {} # skip the pointless `WHERE id IN ()` round-trip on the no-tag-filter hot path
|
||||
out: dict[str, list[int]] = {}
|
||||
for tid, cat in db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids))).all():
|
||||
visible = or_(Tag.user_id.is_(None), Tag.user_id == user.id)
|
||||
for tid, cat in db.execute(
|
||||
select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids), visible)
|
||||
).all():
|
||||
out.setdefault(cat, []).append(tid)
|
||||
known = {tid for ids in out.values() for tid in ids}
|
||||
if missing := [tid for tid in tag_ids if tid not in known]:
|
||||
out[MISSING_TAG_CATEGORY] = missing
|
||||
return out
|
||||
|
||||
|
||||
@@ -264,7 +280,8 @@ def filtered_query(
|
||||
# the queries that surfaced the video > description; see Video.search_vector). YouTube-like
|
||||
# "fuzzy" matching: word-order-independent, multi-word AND, prefix on the word being typed,
|
||||
# accent-insensitive (unaccent_simple). The channel name still matches as a substring so
|
||||
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort.
|
||||
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort, over the
|
||||
# newest RANK_WINDOW matches only (see there).
|
||||
rank_expr = None
|
||||
if q:
|
||||
# Resolve the channels whose NAME matches up front (there are only a few hundred channels, so
|
||||
@@ -302,7 +319,7 @@ def filtered_query(
|
||||
# ALL its selected tags. ⚠️ feed_summary.passes() reimplements this exact rule in Python for the
|
||||
# sidebar count/facets — keep the two in lockstep (equivalence-tested by
|
||||
# test_feed_summary_matches_count_plus_facets).
|
||||
by_category = tags_by_category(db, tags)
|
||||
by_category = tags_by_category(db, user, tags)
|
||||
# Facet counting drops the category being counted so its own chips don't zero each
|
||||
# other out (standard drill-down faceting: a category's count ignores its own
|
||||
# selections but still honours the other categories' filters).
|
||||
@@ -393,7 +410,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
||||
return {"count": 0, "facets": {}}
|
||||
|
||||
# channel -> its visible tag ids, plus each tag's category (for grouping the facet output).
|
||||
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||
visible = _tag_visible(user)
|
||||
channel_tags: dict[str, set[int]] = {}
|
||||
tag_category: dict[int, str] = {}
|
||||
for cid, tid, cat in db.execute(
|
||||
@@ -408,7 +425,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
||||
# base channel carries still filters everything out for its category — matching the SQL `IN (empty)`
|
||||
# behaviour).
|
||||
selected_by_cat: dict[str, set[int]] = {
|
||||
cat: set(ids) for cat, ids in tags_by_category(db, params.get("tags") or []).items()
|
||||
cat: set(ids) for cat, ids in tags_by_category(db, user, params.get("tags") or []).items()
|
||||
}
|
||||
|
||||
def passes(ch_tags: set[int], exclude_cat: str | None) -> bool:
|
||||
@@ -436,7 +453,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
||||
# Drop this category's own selections so its chips keep independent counts; a conjunctive
|
||||
# topic-AND keeps them (each remaining chip narrows to channels that ALSO have the selected).
|
||||
exclude = None if tag_modes.get(category, "or") == "and" else category
|
||||
for cid, ch_tags in channel_tags.items():
|
||||
for ch_tags in channel_tags.values():
|
||||
if not passes(ch_tags, exclude):
|
||||
continue
|
||||
for tid in ch_tags:
|
||||
@@ -446,7 +463,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
||||
|
||||
|
||||
def feed_years(db: Session, user: User, **params) -> list[dict]:
|
||||
"""Per-year video counts for the current filter context (item 8 / E8·S2): powers the year control's
|
||||
"""Per-year video counts for the current filter context: powers the year control's
|
||||
quick chips + the "more…" per-year-counts dialog. Every OTHER filter (tags, channels, content type,
|
||||
search, watch state, scope) still applies, but the publish-date narrowing itself is DROPPED —
|
||||
published_after/before and max_age_days — so the year list is the full set of years available and
|
||||
@@ -478,8 +495,10 @@ def feed_years(db: Session, user: User, **params) -> list[dict]:
|
||||
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
|
||||
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
||||
# (OFFSET would shift and duplicate/skip rows).
|
||||
def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) -> list[dict]:
|
||||
"""`lang` orders the title sorts in the reader's alphabet (`collation`); the other keys ignore it."""
|
||||
pub = {"expr": Video.published_at, "asc": False, "nullable": False, "kind": "dt"}
|
||||
title = collated(Video.title, lang)
|
||||
table: dict[str, list[dict]] = {
|
||||
"newest": [pub],
|
||||
"oldest": [{**pub, "asc": True}],
|
||||
@@ -491,10 +510,8 @@ def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
"duration_asc": [
|
||||
{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}
|
||||
],
|
||||
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
|
||||
"title_desc": [
|
||||
{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}
|
||||
],
|
||||
"title": [{"expr": title, "asc": True, "nullable": True, "kind": "str"}],
|
||||
"title_desc": [{"expr": title, "asc": False, "nullable": True, "kind": "str"}],
|
||||
"subscribers": [
|
||||
{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}
|
||||
],
|
||||
@@ -542,6 +559,54 @@ def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
return table.get(sort) or table["newest"]
|
||||
|
||||
|
||||
# Relevance ranks only the newest RANK_WINDOW matches: ts_rank reads every match's tsvector, and
|
||||
# a broad word matches most of the library ("a" ranked 193k rows in 6.8 s on prod, against 1.1 s to just
|
||||
# find them). The rows past the window keep rank 0, so they follow the ranked ones newest first; a word
|
||||
# that broad has no meaningful relevance order anyway, and a specific one fits the window whole.
|
||||
RANK_WINDOW = 2000
|
||||
|
||||
|
||||
def rank_window_start(db: Session, query: Select) -> datetime | None:
|
||||
"""The published_at the ranked window starts at: the RANK_WINDOW-th newest match, None when fewer
|
||||
match (rank them all). The newest-first walk stops after RANK_WINDOW matches (40 ms for "a")."""
|
||||
return db.scalar(
|
||||
query.with_only_columns(Video.published_at)
|
||||
.order_by(Video.published_at.desc())
|
||||
.offset(RANK_WINDOW - 1)
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
|
||||
def windowed_rank(rank_expr, since: datetime | None):
|
||||
"""`rank_expr` inside the window, 0 past it. CASE evaluates ts_rank only for the rows inside."""
|
||||
if since is None:
|
||||
return rank_expr
|
||||
return case((Video.published_at >= since, rank_expr), else_=0)
|
||||
|
||||
|
||||
def windowed_relevance_rows(
|
||||
db: Session, query: Select, keys: list[dict], window: datetime, cursor_id: str | None, n: int
|
||||
) -> list:
|
||||
"""Up to `n` rows of the windowed relevance order, read as its two halves: the window ranked, then
|
||||
the rest newest first (every window row sorts before every row past it). As one ORDER BY the
|
||||
planner computes the CASE for every match and sorts them all (2.7 s for "a" on prod); split, each
|
||||
half rides an index (80 ms + 5 ms). With a cursor, `keys` carry its values (keyset_predicate)."""
|
||||
pub_key = keys[1] # relevance: [rank, published_at]
|
||||
rows: list = []
|
||||
if cursor_id is None or pub_key["value"] >= window:
|
||||
inside = query.where(Video.published_at >= window).order_by(*order_by(keys))
|
||||
if cursor_id is not None:
|
||||
inside = inside.where(keyset_predicate(keys, cursor_id))
|
||||
rows = list(db.execute(inside.limit(n)).all())
|
||||
if len(rows) < n:
|
||||
newest = [pub_key] # past the window every rank is 0: the order is newest first
|
||||
rest = query.where(Video.published_at < window).order_by(*order_by(newest))
|
||||
if cursor_id is not None:
|
||||
rest = rest.where(keyset_predicate(newest, cursor_id))
|
||||
rows += db.execute(rest.limit(n - len(rows))).all()
|
||||
return rows
|
||||
|
||||
|
||||
def order_by(keys: list[dict]):
|
||||
cols = []
|
||||
for k in keys:
|
||||
@@ -588,26 +653,45 @@ def keyset_predicate(keys: list[dict], cursor_id: str):
|
||||
return rec(0)
|
||||
|
||||
|
||||
def encode_cursor(keys: list[dict], row) -> str:
|
||||
def encode_cursor(keys: list[dict], row, window: datetime | None = None) -> str:
|
||||
"""`window` is the relevance window's start: the cursor carries it, so every page ranks against the
|
||||
window page one did. Recomputed per page, a video ingested mid-scroll would push the window's oldest
|
||||
row to rank 0, behind the cursor, and the reader would see it twice."""
|
||||
vals = []
|
||||
for i, k in enumerate(keys):
|
||||
v = getattr(row, f"_k{i}")
|
||||
vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v)
|
||||
payload = json.dumps({"k": vals, "id": row.id})
|
||||
body: dict = {"k": vals, "id": row.id}
|
||||
if window is not None:
|
||||
body["w"] = window.isoformat()
|
||||
payload = json.dumps(body)
|
||||
return base64.urlsafe_b64encode(payload.encode()).decode()
|
||||
|
||||
|
||||
def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]:
|
||||
def _cursor_dt(v):
|
||||
"""A cursor's timestamp: aware or refused. The server writes only aware ones (timestamptz), and
|
||||
windowed_relevance_rows compares them in Python, where a naive one beside an aware one raises — a
|
||||
500 for a hand-made cursor instead of the 400."""
|
||||
if v is None:
|
||||
return None
|
||||
dt = datetime.fromisoformat(v)
|
||||
if dt.tzinfo is None:
|
||||
raise ValueError("naive cursor timestamp")
|
||||
return dt
|
||||
|
||||
|
||||
def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str, datetime | None]:
|
||||
try:
|
||||
payload = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
raw = payload["k"]
|
||||
if len(raw) != len(keys):
|
||||
raise ValueError("key arity mismatch")
|
||||
vals = [
|
||||
datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v
|
||||
for k, v in zip(keys, raw)
|
||||
]
|
||||
return vals, payload["id"]
|
||||
vals = [_cursor_dt(v) if k["kind"] == "dt" else v for k, v in zip(keys, raw, strict=True)]
|
||||
for k, v in zip(keys, vals, strict=True):
|
||||
if v is None and not k["nullable"]:
|
||||
raise ValueError("null value for a non-nullable key")
|
||||
window = payload.get("w")
|
||||
return vals, payload["id"], _cursor_dt(window) if window else None
|
||||
except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc:
|
||||
# Framework-free: the route handler translates this into an HTTP 400 so the
|
||||
# read-model stays web-layer-agnostic (see routes/feed.get_feed).
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""Bug reports built from debug sessions (E16): the files an issue carries, and where they go —
|
||||
downloaded as a zip, or (S4) sent to a Gitea issue tracker. The session itself — what is recorded
|
||||
"""Bug reports built from debug sessions: the files an issue carries, and where they go —
|
||||
downloaded as a zip, or sent to a Gitea issue tracker. The session itself — what is recorded
|
||||
and tagged — is the core module `app/debugsession.py`."""
|
||||
|
||||
@@ -5,7 +5,7 @@ Gitea upload, so the two cannot drift apart.
|
||||
in their language; the server does not rewrite it);
|
||||
* `client.log` / `server.log` — the session's evidence (`debugsession.bundle`) minus the rows the
|
||||
user took out, one JSON object per line, browser rows and server rows apart. `.log`, not
|
||||
`.jsonl`: Gitea refuses that extension (measured, plans/e16 S0);
|
||||
`.jsonl`: Gitea refuses that extension (measured);
|
||||
* the screenshots, in the order they were taken.
|
||||
|
||||
Every row is already redacted twice (at capture, and again when stored), and nothing here adds a
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""The Gitea issue tracker a debug session's issue goes to (E16 S4) — measured against Gitea 1.27.1
|
||||
before a line of this was written (plans/e16-debug-issue.md, S0).
|
||||
"""The Gitea issue tracker a debug session's issue goes to — measured against Gitea 1.27.1
|
||||
before a line of this was written.
|
||||
|
||||
Configured by an admin (sysconfig `issues_gitea_url` / `_repo` / `_token`, the token encrypted at
|
||||
rest). The repository must be PRIVATE: an issue carries its reporter's log — page titles, filter
|
||||
@@ -79,7 +79,7 @@ def target(db: Session) -> dict | None:
|
||||
@contextmanager
|
||||
def _client(settings: tuple[str, str, str]) -> Iterator[tuple[httpx.Client, str]]:
|
||||
# Takes the settings already read, not the session: reading them HERE opened a transaction one
|
||||
# line after `deliver` had ended one, and held it across the calls below (deep-review round 4).
|
||||
# line after `deliver` had ended one, and held it across the calls below.
|
||||
url, repo, token = settings
|
||||
if not (url and repo and token):
|
||||
raise GiteaError("not_configured")
|
||||
@@ -203,7 +203,7 @@ def deliver(
|
||||
# call, so nothing holds a snapshot (and an idle connection) across minutes of tracker I/O.
|
||||
# The settings are read HERE and handed to `_client`, not read inside it: reading them there
|
||||
# was a SELECT one line after this commit, which reopened the transaction and held it — idle,
|
||||
# with its snapshot pinned — across the repository check and the create (deep-review round 4
|
||||
# with its snapshot pinned — across the repository check and the create (a review
|
||||
# measured it in pg_stat_activity). A comment saying otherwise sat right here.
|
||||
checkpoint()
|
||||
with _client(settings) as (client, repo):
|
||||
@@ -223,7 +223,7 @@ def deliver(
|
||||
)
|
||||
# Built ON the state read at the start so nothing already in it is dropped. The claim
|
||||
# is NOT among those things, whatever this comment used to say: `set_delivery` strips
|
||||
# any `sending` passed to it and stamps the row's own from `token` (round 4 — removing
|
||||
# any `sending` passed to it and stamps the row's own from `token` (removing
|
||||
# the spread left the whole suite green, because the claim never travelled here).
|
||||
state = {
|
||||
**state,
|
||||
@@ -239,7 +239,7 @@ def deliver(
|
||||
# already open on the tracker and this is what would have recorded its link. We
|
||||
# still must not write — another send owns the row and is opening its own issue —
|
||||
# so say which issue was left behind, because nothing else will.
|
||||
# noqa TRY400: the traceback is one SendClaimLost carrying the session id and
|
||||
# TRY400: the traceback is one SendClaimLost carrying the session id and
|
||||
# nothing else; the issue number below is the whole news.
|
||||
log.error( # noqa: TRY400
|
||||
"an issue was created by a send that had already lost its claim",
|
||||
|
||||
+15
-15
@@ -10,7 +10,7 @@ the logs were both unjoinable and leaky. `configure_logging()` replaces all four
|
||||
Event shape — the OpenTelemetry log data model, without the SDK; one JSON object per line:
|
||||
ts · level · sev · source · logger · event · msg · attrs
|
||||
plus whatever correlation the context carries (`request_id`, `user_id`, `debug_session_id`,
|
||||
`job_id`) and the app version. That shape is what E22.6's admin console filters on and what a Gitea
|
||||
`job_id`) and the app version. That shape is what the admin log console filters on and what a Gitea
|
||||
issue attachment will carry, so it is deliberately flat and stable.
|
||||
|
||||
Call sites stay ordinary stdlib logging:
|
||||
@@ -106,7 +106,7 @@ class _RequestScope:
|
||||
self.user_id: int | None = None
|
||||
# The user's consented diagnostics level as a logging number; None at the base level.
|
||||
self.level: int | None = None
|
||||
# The user's own open debug session this request named (E16), else None.
|
||||
# The user's own open debug session this request named, else None.
|
||||
self.debug_session_id: str | None = None
|
||||
|
||||
|
||||
@@ -122,11 +122,11 @@ def set_user(
|
||||
debug_session: str | None = None,
|
||||
) -> None:
|
||||
"""Attribute the rest of this request's log lines to `user_id`, and log them at the user's
|
||||
consented diagnostics `level` until `until` (E22 S5; "INFO" = no elevation). Called from the
|
||||
consented diagnostics `level` until `until` ("INFO" = no elevation). Called from the
|
||||
auth dependency — the one place the session is actually resolved into an account — rather than
|
||||
from the request middleware, which runs OUTSIDE the session middleware.
|
||||
|
||||
`debug_session` tags every line with the user's open debug session (E16). The caller has
|
||||
`debug_session` tags every line with the user's open debug session. The caller has
|
||||
already checked it is theirs and open; a session's level arrives here as `level`/`until`.
|
||||
|
||||
The root logger's floor is tracked per elevation: a request without the session's header (an
|
||||
@@ -166,7 +166,7 @@ def bind(**fields: Any):
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for key, token in zip(keys, tokens):
|
||||
for key, token in zip(keys, tokens, strict=True):
|
||||
_CONTEXT_VARS[key].reset(token)
|
||||
|
||||
|
||||
@@ -334,7 +334,7 @@ _RECORD_STANDARD = frozenset(
|
||||
class ContextFilter(logging.Filter):
|
||||
"""Copies the correlation contextvars onto the record. A Filter (not the formatter) so the
|
||||
values are captured at emit time, before a QueueHandler can move the record to another
|
||||
thread — the DB sink in S3 depends on that."""
|
||||
thread — the DB sink depends on that."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
for name, var in _CONTEXT_VARS.items():
|
||||
@@ -355,7 +355,7 @@ class ContextFilter(logging.Filter):
|
||||
|
||||
class LevelGate(logging.Filter):
|
||||
"""The configured level, enforced at the HANDLER, with one exception: the lines of a request
|
||||
whose user raised their diagnostics level (E22 S5).
|
||||
whose user raised their diagnostics level.
|
||||
|
||||
The root logger's level cannot do this alone: it is one number for the whole process, and the
|
||||
point is to log ONE user's requests in detail while everyone else's stay at the base level. So
|
||||
@@ -457,7 +457,7 @@ class TextFormatter(logging.Formatter):
|
||||
# --- configuration -------------------------------------------------------------------------
|
||||
# Third-party loggers that are noisy at our own level. Pinned explicitly so the diagnostics knob can
|
||||
# lower the ROOT level to DEBUG/TRACE without drowning the output in SQL statements and HTTP chatter
|
||||
# — raising THESE is a deliberate, separate act (E22.6's console).
|
||||
# — raising THESE is a deliberate, separate act (the admin log console).
|
||||
_THIRD_PARTY_LEVELS = {
|
||||
"sqlalchemy": "WARNING",
|
||||
"sqlalchemy.engine": "WARNING",
|
||||
@@ -466,7 +466,6 @@ _THIRD_PARTY_LEVELS = {
|
||||
"httpx": "WARNING",
|
||||
"httpcore": "WARNING",
|
||||
"asyncio": "WARNING",
|
||||
"multipart": "WARNING",
|
||||
"PIL": "WARNING",
|
||||
"alembic": "INFO",
|
||||
# uvicorn's own startup/shutdown/WebSocket lines are worth keeping; its ACCESS log is not —
|
||||
@@ -482,7 +481,7 @@ def get_logger(name: str) -> logging.Logger:
|
||||
`siftlode.plex.sync`.
|
||||
|
||||
The names were literals before, and 25 of them covered 34 modules — six different Plex modules
|
||||
all called themselves `siftlode.plex`, so neither a level pinned per logger nor E22.6's console
|
||||
all called themselves `siftlode.plex`, so neither a level pinned per logger nor the console's
|
||||
filter could tell a sync line from a stream line. Derived from `__name__` there is exactly one
|
||||
name per module, it cannot drift from the file it sits in, and `test_logger_names.py` fails if a
|
||||
module hand-rolls one again.
|
||||
@@ -563,12 +562,12 @@ def current_level() -> str:
|
||||
return logging.getLevelName(_base_level or logging.getLogger().getEffectiveLevel())
|
||||
|
||||
|
||||
# --- per-user elevation (E22 S5) -----------------------------------------------------------
|
||||
# --- per-user elevation -----------------------------------------------------------
|
||||
# The base level (`LOG_LEVEL`, then the admin knob). NOTSET until configure_logging runs — a unit
|
||||
# test, a script — so the gate lets everything through there, as it did before the gate existed.
|
||||
_base_level = logging.NOTSET
|
||||
# → (level number, expiry) for every elevation this process has seen and not yet expired: a user's
|
||||
# own, by user id, and a debug session's (E16), by (user id, session id) — its requests alone carry
|
||||
# own, by user id, and a debug session's, by (user id, session id) — its requests alone carry
|
||||
# its level, so it is tracked apart from the user's own requests.
|
||||
_ElevationKey = int | tuple[int, str]
|
||||
_elevations: dict[_ElevationKey, tuple[int, datetime]] = {}
|
||||
@@ -668,7 +667,7 @@ def current_source() -> str:
|
||||
|
||||
|
||||
# --- request logging -----------------------------------------------------------------------
|
||||
# A client may supply its own request id so a browser log line and a server line join up (E22 S4).
|
||||
# A client may supply its own request id so a browser log line and a server line join up.
|
||||
# Accepted only in this shape: it ends up in a log field, and an unbounded header would be a way to
|
||||
# write arbitrary text into the log.
|
||||
_REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,64}$")
|
||||
@@ -689,10 +688,11 @@ _POLLED_ROUTES = frozenset(
|
||||
{
|
||||
"/api/me",
|
||||
"/api/me/notifications/unread_count",
|
||||
"/api/messages/unread_count",
|
||||
"/api/downloads",
|
||||
"/api/downloads/index",
|
||||
"/api/downloads/usage",
|
||||
"/api/files",
|
||||
"/api/files/folders",
|
||||
"/api/files/usage",
|
||||
"/api/sync/my-status",
|
||||
"/api/admin/scheduler",
|
||||
"/api/admin/downloads",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The `log_events` sink (E22 S3): the stored copy of the log the diagnostics console reads.
|
||||
"""The `log_events` sink: the stored copy of the log the diagnostics console reads.
|
||||
|
||||
stdout stays the complete log (docker keeps 3×50 MB of it); this table keeps the part worth querying
|
||||
for `log_retention_days`:
|
||||
@@ -6,7 +6,7 @@ for `log_retention_days`:
|
||||
* every WARNING and above,
|
||||
* the `http.request` line of every request at INFO or above (not the DEBUG-level health checks and
|
||||
static files),
|
||||
* every line of a request whose user raised their diagnostics level (E22 S5 — `logs.LevelGate`
|
||||
* every line of a request whose user raised their diagnostics level (`logs.LevelGate`
|
||||
marks them), which is the detail they consented to have kept,
|
||||
* nothing a caller marked `extra={logs.PERSIST: False}` — how the request log caps anonymous
|
||||
traffic per client.
|
||||
@@ -347,8 +347,8 @@ def gc(db: Session, retention_days: int) -> int:
|
||||
|
||||
|
||||
def serialize(row: LogEvent) -> dict:
|
||||
"""A stored row as the API sends it — one shape for every reader: the admin console (E22.6) and
|
||||
a debug session's evidence list (E16)."""
|
||||
"""A stored row as the API sends it — one shape for every reader: the admin console and
|
||||
a debug session's evidence list."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"ts": iso(row.ts),
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Bilingual text for every outbound mail, and the one resolver that decides which language a
|
||||
recipient gets (E9.5 S7).
|
||||
recipient gets.
|
||||
|
||||
Why a catalog instead of literals in `email.py`: HU/EN parity is a house rule for every user-facing
|
||||
string, and a mail is the one user-facing surface the reader CANNOT re-render by switching the UI
|
||||
@@ -53,7 +53,7 @@ def lang_for_user(user: User | None) -> str:
|
||||
|
||||
def lang_for_email(db: Session, email: str) -> str:
|
||||
"""The language for an address that may not have an account yet: their own account preference,
|
||||
else the language they filled our access-request form in, else English.
|
||||
else the language their access request came in (the locale Google reported), else English.
|
||||
|
||||
Matched CANONICALLY like every other identity lookup in this epic, and it walks past rows that
|
||||
carry no usable code instead of stopping at the first one — prod really does hold two accounts
|
||||
@@ -82,8 +82,8 @@ def lang_for_email(db: Session, email: str) -> str:
|
||||
# a name, not a translated string, and giving them keys would invite someone to "translate" them.
|
||||
MAIL_TEXT: dict[str, dict[str, str]] = {
|
||||
# ── How long a link lasts ───────────────────────────────────────────────────────────────
|
||||
# Rendered, not interpolated as a raw hour count: the approval mail's token lives seven days
|
||||
# (E9.5 S6), and "expires in 168 hours" is a unit no reader converts.
|
||||
# Rendered, not interpolated as a raw hour count: the approval mail's token lives seven days,
|
||||
# and "expires in 168 hours" is a unit no reader converts.
|
||||
"duration.hours": {"en": "{n} hours", "hu": "{n} óra"},
|
||||
"duration.days": {"en": "{n} days", "hu": "{n} nap"},
|
||||
# ── Shared closing fragments ────────────────────────────────────────────────────────────────
|
||||
|
||||
+105
-45
@@ -1,36 +1,38 @@
|
||||
import asyncio
|
||||
import math
|
||||
import mimetypes
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
|
||||
# Logging is configured HERE, before the route modules are imported, so every logger they create at
|
||||
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
||||
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
||||
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
||||
from app import logs
|
||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||
|
||||
configure_logging("api")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.exc import DataError
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from app import logsink
|
||||
from app import state
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.plex import stream as plex_stream
|
||||
from app.plex import subtitles as plex_subtitles
|
||||
from app.routes import (
|
||||
# Logging is configured HERE, before the app's own modules are imported, so a line one of them logs
|
||||
# WHILE importing already has the format (a logger is configured whenever it is created —
|
||||
# app/logs.py — but a line emitted before `configure_logging` goes out raw). ONE format for the api,
|
||||
# the worker, the download child and alembic alike — this replaces the old uvicorn --log-config +
|
||||
# the hand-rolled fallback that used to live here, and uvicorn's access log is replaced by the
|
||||
# `http.request` middleware below. The imports after it are E402 on purpose, each one marked.
|
||||
from app import logs
|
||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||
|
||||
configure_logging("api")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first
|
||||
from app.config import settings # noqa: E402 — logging is configured first
|
||||
from app.db import SessionLocal, engine # noqa: E402 — logging is configured first
|
||||
from app.downloads import service as download_service # noqa: E402 — logging is configured first
|
||||
from app.downloads import storage # noqa: E402 — logging is configured first
|
||||
from app.plex import stream as plex_stream # noqa: E402 — logging is configured first
|
||||
from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first
|
||||
from app.routes import ( # noqa: E402 — logging is configured first
|
||||
admin,
|
||||
admin_logs as admin_logs_routes,
|
||||
audit as audit_routes,
|
||||
@@ -42,9 +44,9 @@ from app.routes import (
|
||||
discover as discover_routes,
|
||||
downloads,
|
||||
feed,
|
||||
files as files_routes,
|
||||
health,
|
||||
me,
|
||||
messages,
|
||||
notifications,
|
||||
playlists,
|
||||
plex as plex_routes,
|
||||
@@ -56,14 +58,19 @@ from app.routes import (
|
||||
setup as setup_routes,
|
||||
sync,
|
||||
tags,
|
||||
tokens as tokens_routes,
|
||||
version,
|
||||
)
|
||||
from app.scheduler import shutdown_scheduler, start_scheduler
|
||||
from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 — logging is configured first
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
log.info("Siftlode starting up")
|
||||
problem = storage.files_root_problem(settings.files_root)
|
||||
if problem:
|
||||
# The rest of the app still serves; the files library cannot (the worker refuses to start).
|
||||
log.critical("The files library is not reachable: %s. See docs/self-hosting.md.", problem)
|
||||
# The stored copy of the log (log_events). The entrypoint has already migrated the schema.
|
||||
logsink.start()
|
||||
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
||||
@@ -75,10 +82,22 @@ async def lifespan(app: FastAPI):
|
||||
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
||||
try:
|
||||
set_level(sysconfig.get_str(db, "log_level"))
|
||||
except Exception: # noqa: BLE001 — any DB/config failure keeps the bootstrap level
|
||||
except Exception: # any DB/config failure keeps the bootstrap level
|
||||
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
||||
# and `%s` of the exception does not say which of them raised.
|
||||
log.exception("could not read the stored log level; staying at %s", current_level())
|
||||
if not problem:
|
||||
try:
|
||||
left = download_service.library_left_behind(db, settings.files_root)
|
||||
except Exception: # a database not reachable yet: the check is a hint, never a gate
|
||||
log.exception("could not check the files library against its database")
|
||||
else:
|
||||
if left:
|
||||
log.critical(
|
||||
"The files library is empty but the database names files: the volume "
|
||||
"mounted at %s is not the one that holds them. See docs/self-hosting.md.",
|
||||
settings.files_root,
|
||||
)
|
||||
if not state.is_configured(db):
|
||||
token = state.rotate_setup_token(db)
|
||||
url = f"{settings.app_base}/setup?token={token}"
|
||||
@@ -96,6 +115,9 @@ async def lifespan(app: FastAPI):
|
||||
# owns, so racing it against early playback is safe.
|
||||
sweep = asyncio.create_task(_sweep_hls_orphans())
|
||||
reaper = asyncio.create_task(_reap_hls_sessions())
|
||||
# Fire and forget as well: a rebuild after an ICU upgrade takes seconds, and the title sorts serve
|
||||
# meanwhile (CONCURRENTLY) in the old index's order, as they did before the restart.
|
||||
heal = asyncio.create_task(_heal_collation_drift())
|
||||
start_scheduler()
|
||||
try:
|
||||
yield
|
||||
@@ -105,7 +127,7 @@ async def lifespan(app: FastAPI):
|
||||
# pending ("Task was destroyed but it is pending!" — noise that reads like a bug during
|
||||
# triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own
|
||||
# bookkeeping honest.
|
||||
for task in (sweep, reaper):
|
||||
for task in (sweep, reaper, heal):
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
@@ -129,7 +151,7 @@ async def _reap_hls_sessions() -> None:
|
||||
A FINISHED session is reaped on the short rule here, not kept for the full idle window: an open
|
||||
player checkpoints every 10 s and that route touches the session, so a paused viewer is no longer
|
||||
invisible to this pass (see `reap_idle`). Keeping every finished session for ten minutes instead
|
||||
held one whole segmented copy of a film per item on the download volume (round 5)."""
|
||||
held one whole segmented copy of a film per item on the download volume."""
|
||||
while True:
|
||||
await asyncio.sleep(_HLS_REAP_EVERY_S)
|
||||
try:
|
||||
@@ -138,6 +160,18 @@ async def _reap_hls_sessions() -> None:
|
||||
log.exception("HLS session reap failed")
|
||||
|
||||
|
||||
async def _heal_collation_drift() -> None:
|
||||
"""Background startup check of the ICU collation the title-sort indexes are built in (see
|
||||
`collation.heal_drift`). A failure is logged, never fatal: the sorts still answer, only possibly in
|
||||
the old order."""
|
||||
try:
|
||||
await asyncio.to_thread(collation.heal_drift, engine)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("ICU collation drift check failed")
|
||||
|
||||
|
||||
async def _sweep_hls_orphans() -> None:
|
||||
"""Background half of the startup HLS sweep (see lifespan). Owns its own errors: a failure here
|
||||
must never take the app down, and a cancelled shutdown is not one."""
|
||||
@@ -173,8 +207,8 @@ async def _nul_in_text(request: Request, exc: DataError) -> JSONResponse:
|
||||
NUL in text, so the driver refuses to even send the query — and a `%00` in a query parameter or
|
||||
an escaped NUL in a JSON body (a playlist's `contains=`, a reorder's key list, a collection
|
||||
search) was a 500 with a traceback wherever it reached a comparison. Guarding each site did not
|
||||
hold: round 3 guarded the by-rating_key lookup and missed the rest. Any other DataError is left
|
||||
alone — re-raised, it stays the 500 it always was."""
|
||||
hold: an earlier fix guarded the by-rating_key lookup and missed the rest. Any other DataError
|
||||
is left alone — re-raised, it stays the 500 it always was."""
|
||||
if "NUL (0x00)" not in str(exc.orig):
|
||||
raise exc
|
||||
return JSONResponse(
|
||||
@@ -189,15 +223,6 @@ app.add_middleware(
|
||||
https_only=settings.session_https_only, # Secure flag when served over HTTPS (prod)
|
||||
)
|
||||
|
||||
if settings.frontend_origin:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[settings.frontend_origin],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Paths reachable while the instance is unconfigured (setup mode). Everything else under /api or
|
||||
# /auth is locked until the wizard finishes; static/SPA always loads so the wizard page can render.
|
||||
_SETUP_OPEN_PREFIXES = ("/api/setup", "/api/version")
|
||||
@@ -217,6 +242,34 @@ async def setup_gate(request, call_next):
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# SEC-20 — the global request-body cap (app/bodylimit.py). FastAPI reads a declared body IN FULL
|
||||
# before the first dependency, so without this the anonymous login route is a free memory/time sink.
|
||||
#
|
||||
# WHERE it sits is the whole design, and it is pinned by `test_body_limit.py` rather than by this
|
||||
# comment, because `add_middleware` inverts the reading order (last registered = OUTERMOST) and a
|
||||
# stack described in prose drifts. Registered here it lands directly INSIDE CORS and directly
|
||||
# OUTSIDE the setup gate, which is the narrowest position that satisfies both constraints:
|
||||
#
|
||||
# * inside CORS and the security headers, so the 413 is decorated like every other response. A
|
||||
# refusal the browser drops for a missing Access-Control-Allow-Origin is no better than a hang,
|
||||
# and `capped_body`'s 413 one layer down IS decorated — two shapes for one fact is exactly what
|
||||
# bodylimit.py exists to avoid.
|
||||
# * outside the setup gate, the session and every auth dependency under the router — none of which
|
||||
# run for a body that is already too large. The setup gate matters specifically: on an instance
|
||||
# that is not yet configured it opens a DB session per request, which is precisely the anonymous
|
||||
# pre-auth work SEC-20 exists to remove.
|
||||
app.add_middleware(bodylimit.BodyLimitMiddleware)
|
||||
|
||||
if settings.frontend_origin:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[settings.frontend_origin],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def static_cache_headers(request, call_next):
|
||||
"""Long-cache the content-hashed SPA bundles. Vite hashes their filename, so a given
|
||||
@@ -234,7 +287,8 @@ async def static_cache_headers(request, call_next):
|
||||
# Baseline security response headers (SEC-1). A minimal, safe set that needs no per-route tuning:
|
||||
# nosniff stops MIME-confusion, the Referrer-Policy keeps the (private) app URL from leaking to
|
||||
# external sites the user follows, `frame-ancestors 'none'` blocks click-jacking by refusing to be
|
||||
# iframed anywhere (we never embed our own pages; we embed OTHERS, which this does not affect), and
|
||||
# iframed anywhere (we embed OTHERS, which this does not affect; the one own answer the app frames, a
|
||||
# library file's preview, sets its own `frame-ancestors 'self'` — these are defaults, `setdefault`), and
|
||||
# Permissions-Policy turns off powerful features the app never uses. `X-Frame-Options: DENY` duplicates
|
||||
# the frame-ancestors intent for legacy engines that never implemented CSP frame-ancestors (belt and
|
||||
# suspenders — modern browsers use the CSP directive and ignore the header). A full Content-Security-Policy
|
||||
@@ -259,7 +313,8 @@ async def security_headers(request, call_next):
|
||||
|
||||
# The access log (app/logs.py): request id, route template, status, duration — one line, never a
|
||||
# query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole
|
||||
# request, including what the three above do. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# request, including what every layer above does — the body cap included, so a 413 is still logged
|
||||
# with its request id and duration. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# which is why the account comes from `request.state.user_id` (set by the auth dependency, the one
|
||||
# place a session is actually resolved into an account) rather than from `request.session`.
|
||||
app.middleware("http")(log_requests)
|
||||
@@ -276,13 +331,14 @@ app.include_router(me.router)
|
||||
app.include_router(diagnostics_routes.router)
|
||||
app.include_router(debug_sessions_routes.router)
|
||||
app.include_router(notifications.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(channels.router)
|
||||
app.include_router(playlists.router)
|
||||
app.include_router(saved_views.router)
|
||||
app.include_router(plex_routes.router)
|
||||
app.include_router(downloads.router)
|
||||
app.include_router(downloads.admin_router)
|
||||
app.include_router(files_routes.router)
|
||||
app.include_router(tokens_routes.router)
|
||||
app.include_router(public_routes.router)
|
||||
app.include_router(admin.router)
|
||||
app.include_router(config_routes.router)
|
||||
@@ -295,8 +351,6 @@ app.include_router(setup_routes.router)
|
||||
|
||||
# Ensure modern image types resolve to the right Content-Type when FileResponse guesses from the
|
||||
# filename (the runtime's mimetypes db doesn't always know .webp → it'd fall back to octet-stream).
|
||||
import mimetypes
|
||||
|
||||
mimetypes.add_type("image/webp", ".webp")
|
||||
mimetypes.add_type("image/avif", ".avif")
|
||||
|
||||
@@ -308,6 +362,8 @@ STATIC_DIR = Path(__file__).parent / "static_spa"
|
||||
# stay cached but forces revalidation (cheap 304 via the FileResponse ETag) on every load. The
|
||||
# hashed bundles under /assets can cache forever (a new build changes their filename).
|
||||
INDEX_HTML = STATIC_DIR / "index.html"
|
||||
# A public share link's own page: the player alone, without the app's bundle (frontend/watch.html).
|
||||
WATCH_HTML = STATIC_DIR / "watch.html"
|
||||
INDEX_HEADERS = {"Cache-Control": "no-cache"}
|
||||
app.mount(
|
||||
"/assets",
|
||||
@@ -323,16 +379,16 @@ async def index() -> FileResponse:
|
||||
|
||||
@app.get("/watch/{token}")
|
||||
async def watch_page(token: str):
|
||||
"""Serve the SPA for a public share link, but with per-video Open Graph tags injected so the
|
||||
"""Serve the public player page for a share link, with per-video Open Graph tags injected so the
|
||||
link unfurls richly (title/channel/thumbnail) in Messenger/social crawlers. A real browser
|
||||
ignores the extra tags and hydrates WatchPage as usual; invalid/expired/password links fall
|
||||
back to the generic card."""
|
||||
ignores the extra tags and boots WatchPage; invalid/expired/password links fall back to the
|
||||
generic card."""
|
||||
from app.downloads import og
|
||||
|
||||
with SessionLocal() as db:
|
||||
rendered = og.render_watch_html(token, INDEX_HTML, db)
|
||||
rendered = og.render_watch_html(token, WATCH_HTML, db)
|
||||
if rendered is None:
|
||||
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|
||||
return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
|
||||
return HTMLResponse(rendered, headers=INDEX_HEADERS)
|
||||
|
||||
|
||||
@@ -350,4 +406,8 @@ async def spa_fallback(full_path: str) -> FileResponse:
|
||||
# Real SPA-root assets (welcome images, favicon, robots.txt) rarely change and have
|
||||
# stable names, so a moderate cache is safe and satisfies the cache-lifetime audit.
|
||||
return FileResponse(candidate, headers={"Cache-Control": "public, max-age=604800"})
|
||||
# A /watch/ path the route above does not match (a trailing segment, no token) is still the
|
||||
# player's: WatchPage reads the token off the path and says when a link is gone.
|
||||
if full_path.startswith("watch/"):
|
||||
return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
|
||||
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|
||||
|
||||
+258
-143
@@ -1,4 +1,4 @@
|
||||
from datetime import date, datetime
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
@@ -18,6 +18,7 @@ from sqlalchemy import (
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
@@ -25,6 +26,32 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
from app.db import Base
|
||||
from app.utils import canonical_email
|
||||
|
||||
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
||||
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
||||
# `User.display_name` (their Google profile name), `DownloadJob.display_name` and
|
||||
# `DownloadJob.display_uploader`. A value wider than the column is a psycopg DataError, which
|
||||
# `main.py` re-raises for everything but the NUL case — i.e. a bare 500 for a name somebody pasted.
|
||||
DISPLAY_TEXT_MAX = 255
|
||||
|
||||
|
||||
def display_text(raw: str | None) -> str | None:
|
||||
"""A user-supplied display string on its way into one of those columns: stripped, truncated,
|
||||
blank -> None.
|
||||
|
||||
One definition because there are SIX writers and they had drifted: `POST /api/downloads`,
|
||||
`POST /api/downloads/edit`, `POST /api/files/upload`, the library PATCH's two fields, and
|
||||
the Google sign-in callback. Only the PATCH truncated. Truncating is the right answer rather
|
||||
than refusing: the name is a label on work the user has already done — a file already
|
||||
transferred, an account already authenticated — and losing its tail is not worth failing the
|
||||
request over.
|
||||
|
||||
`test_upload.py` holds both halves: that every route assigning one of these fields goes through
|
||||
this helper (parsed, not grepped), and that each of those routes truncates rather than 500s. A
|
||||
sweep is exactly the kind of thing that gets done five-sixths of the way, which is how this
|
||||
ended up being written twice.
|
||||
"""
|
||||
return (raw or "").strip()[:DISPLAY_TEXT_MAX] or None
|
||||
|
||||
|
||||
# The resume-position columns (VideoState / PlexState.position_seconds) are int4. The checkpoint
|
||||
# routes bound their bodies by this, so an out-of-range position is a 422 rather than a 500.
|
||||
@@ -33,7 +60,7 @@ POSITION_MAX_S = 2**31 - 1
|
||||
|
||||
class TimestampMixin:
|
||||
"""Adds a server-defaulted `created_at`. Models that index created_at or order it
|
||||
differently (QuotaEvent, Notification, Message) keep their own column on purpose."""
|
||||
differently (QuotaEvent, Notification) keep their own column on purpose."""
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
@@ -75,15 +102,15 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
# Google sign-in that failed to find it), and a migration must not merge accounts on its own.
|
||||
email_canonical: Mapped[str] = mapped_column(String(320), index=True, server_default="")
|
||||
# The address Google asserted for this account, recorded at sign-in / link time. NOT the same
|
||||
# thing as `email`: since E9.5 S3 a Google sign-in no longer overwrites the address the user
|
||||
# thing as `email`: a Google sign-in no longer overwrites the address the user
|
||||
# registered with, so a deliberate `me+siftlode@gmail.com` survives — and this is where the other
|
||||
# half of the truth lives, for the UI to show when the two differ. NULL until the next Google
|
||||
# sign-in for accounts linked before this column existed.
|
||||
google_email: Mapped[str | None] = mapped_column(String(320))
|
||||
# argon2id hash for email+password login; NULL for Google-only / demo accounts.
|
||||
# argon2id hash for email+password login; NULL for Google-only accounts.
|
||||
password_hash: Mapped[str | None] = mapped_column(String(255))
|
||||
# When SEC-10 dropped a preset password on Google adoption (see the single caller of
|
||||
# auth.revoke_preset_password_if_unverified; admin approval no longer revokes — E9.5 S1). Without
|
||||
# auth.revoke_untrusted_preset_password; admin approval no longer revokes). Without
|
||||
# this trace a revoked account was indistinguishable from a Google-only one, so it failed the
|
||||
# password-reset eligibility check and could never be recovered — it could neither sign in nor
|
||||
# reset. Set = "had a password, lost it to the guard" -> reset-eligible. Cleared whenever a
|
||||
@@ -97,7 +124,8 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
# Admin-imposed access block, distinct from is_active (approval lifecycle). A suspended
|
||||
# account can't sign in by any method and its live sessions are rejected; the user is told
|
||||
# why and pointed at the operator's contact. The demo account is never suspendable.
|
||||
# why and pointed at the operator's contact. The worker leaves its queued downloads unclaimed
|
||||
# until the block is lifted (`worker._claim_job`).
|
||||
is_suspended: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Server-side session revocation (SA4): a monotonic counter bumped on password reset, password
|
||||
# change, or "log out everywhere". A signed session cookie records the epoch it was minted under;
|
||||
@@ -106,26 +134,20 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
session_epoch: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# When this account last signed in (Google or password), stamped by the one sign-in path,
|
||||
# routes/auth._establish_session. NULL = not since the column existed (0085; no backfill is
|
||||
# possible). The shared demo account is never stamped — its "last login" would mean nobody.
|
||||
# possible).
|
||||
last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The diagnostics level this user consented to (E22 S5) and when it lapses back to the base
|
||||
# The diagnostics level this user consented to and when it lapses back to the base
|
||||
# level. Both NULL = base. Written only by `PUT /api/me/diagnostics`, which computes the expiry
|
||||
# itself; read through `app.diagnostics.effective_level`, never directly (an expired level is
|
||||
# not cleared eagerly — it simply stops counting).
|
||||
diagnostics_level: Mapped[str | None] = mapped_column(String(8))
|
||||
diagnostics_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
avatar_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
role: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
# The user's own YouTube channel id, resolved lazily (channels.list?mine=true) and
|
||||
# cached. Used to exclude their own channel from playlist discovery. NULL = unresolved.
|
||||
yt_channel_id: Mapped[str | None] = mapped_column(String(32))
|
||||
# The single shared "demo" account: signed into without Google OAuth (via a whitelisted
|
||||
# email on the login page), has no OAuth token / YouTube scope, and its per-user state is
|
||||
# shared by everyone who enters this way. Exactly one such row is expected.
|
||||
is_demo: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
# Free-form UI preferences (theme, color scheme, font scale, default filters…).
|
||||
preferences: Mapped[dict | None] = mapped_column(JSON)
|
||||
|
||||
@@ -192,25 +214,12 @@ class Invite(Base, CanonicalEmailMixin):
|
||||
# The language this person asked us in ("en"/"hu"), captured when the request was made: the page
|
||||
# their form was on, or the locale Google reported for a refused sign-in. NULL for a row the
|
||||
# admin whitelisted by hand — nobody has told us anything about that mailbox, which is exactly
|
||||
# why the invitation send asks (E9.5 S7). Captured rather than derived for the same reason
|
||||
# why the invitation send asks. Captured rather than derived for the same reason
|
||||
# `preferences.language` and `google_email` are: the signal exists only in that one moment and
|
||||
# cannot be backfilled afterwards.
|
||||
lang: Mapped[str | None] = mapped_column(String(8))
|
||||
|
||||
|
||||
class DemoWhitelist(Base, TimestampMixin):
|
||||
"""Emails that may enter the shared demo account from the login page (no Google OAuth).
|
||||
A hidden, admin-curated list of "keys to the same door" — every entry logs into the one
|
||||
shared demo user. Distinct from Invite (which gates real Google sign-in)."""
|
||||
|
||||
__tablename__ = "demo_whitelist"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
||||
note: Mapped[str | None] = mapped_column(Text)
|
||||
added_by: Mapped[str | None] = mapped_column(String(320)) # admin email
|
||||
|
||||
|
||||
class Channel(Base, TimestampMixin):
|
||||
"""A YouTube channel. Shared across all users (one channel's videos are the same
|
||||
for everyone), so its expensive metadata is fetched and stored only once."""
|
||||
@@ -302,10 +311,12 @@ class Video(Base, TimestampMixin):
|
||||
__tablename__ = "videos"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
||||
# No standalone channel_id index: the composite `ix_videos_channel_pub_desc`
|
||||
# (channel_id, published_at DESC NULLS LAST, id) from migration 0067 leads with channel_id, so it
|
||||
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) AND the per-channel feed
|
||||
# sort. Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
||||
# No standalone channel_id index: the composite `ix_videos_channel_pub_cover`
|
||||
# (channel_id, published_at DESC NULLS LAST, id) from migration 0095 leads with channel_id, so it
|
||||
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) and the per-channel feed
|
||||
# sort. Its INCLUDEd columns make the channel manager's sums and the feed sidebar's counts Index Only
|
||||
# Scans (no heap): a column those queries start reading belongs in it (test_channel_list_index_only).
|
||||
# Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
||||
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
||||
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
||||
@@ -319,6 +330,9 @@ class Video(Base, TimestampMixin):
|
||||
DateTime(timezone=True), index=True, nullable=False, server_default=func.now()
|
||||
)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
|
||||
# ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to
|
||||
# `feedquery.order_by` by test_feed_sort_indexes.
|
||||
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
@@ -450,6 +464,27 @@ class VideoState(Base, UpdatedAtMixin):
|
||||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class LibraryState(Base):
|
||||
"""Where a user stopped in a library item (their own download, or one shared with them) — the
|
||||
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
|
||||
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
|
||||
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
|
||||
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
|
||||
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
|
||||
|
||||
__tablename__ = "library_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class SearchFind(Base, TimestampMixin):
|
||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||
|
||||
@@ -555,7 +590,7 @@ class AuditLog(Base):
|
||||
|
||||
|
||||
class LogEvent(Base):
|
||||
"""One stored log event (E22 S3) — the `log_events` table the diagnostics console reads. Rows are
|
||||
"""One stored log event — the `log_events` table the diagnostics console reads. Rows are
|
||||
written by app/logsink.py's background writer, never inside a request's transaction, already
|
||||
redacted (the same `logs.record_fields` the stdout line is built from).
|
||||
|
||||
@@ -568,7 +603,7 @@ class LogEvent(Base):
|
||||
__tablename__ = "log_events"
|
||||
__table_args__ = (
|
||||
Index("ix_log_events_user_ts", "user_id", "ts"),
|
||||
# E16: an issue's bundle is `WHERE debug_session_id = …`; nearly every row has none.
|
||||
# An issue's bundle is `WHERE debug_session_id = …`; nearly every row has none.
|
||||
Index(
|
||||
"ix_log_events_debug_session",
|
||||
"debug_session_id",
|
||||
@@ -594,7 +629,7 @@ class LogEvent(Base):
|
||||
|
||||
|
||||
class DebugSession(Base):
|
||||
"""One recording: Start → reproduce → Stop (E16, app/debugsession.py). While it is open, every
|
||||
"""One recording: Start → reproduce → Stop (app/debugsession.py). While it is open, every
|
||||
request of its user that names it (`X-Debug-Session`) is logged at `level` and tagged with `id`,
|
||||
and the issue it ends in is built from the rows carrying that tag. `until` is the hard cap the
|
||||
server set at Start; an open session past it is over whether or not the sweep has closed it yet.
|
||||
@@ -619,12 +654,12 @@ class DebugSession(Base):
|
||||
# app/debugsession.py `EndReason`: stopped by the user, or past `until`.
|
||||
ended_reason: Mapped[str | None] = mapped_column(String(16))
|
||||
issue_url: Mapped[str | None] = mapped_column(String(512))
|
||||
# The issue being written from it (E16 S3; app/debugsession.py `save_draft` holds its shape).
|
||||
# The issue being written from it (app/debugsession.py `save_draft` holds its shape).
|
||||
draft: Mapped[dict | None] = mapped_column(JSONB)
|
||||
|
||||
|
||||
class DebugScreenshot(Base):
|
||||
"""A screenshot attached to a debug session (E16) — pasted, dropped, or captured from the pill.
|
||||
"""A screenshot attached to a debug session — pasted, dropped, or captured from the pill.
|
||||
Stored in the row: bounded per session (app/debugsession.py), and gone with it by cascade."""
|
||||
|
||||
__tablename__ = "debug_screenshots"
|
||||
@@ -783,44 +818,6 @@ class Notification(Base):
|
||||
)
|
||||
|
||||
|
||||
class Message(Base):
|
||||
"""A message in the notification module (phase 2).
|
||||
|
||||
Two kinds share this table:
|
||||
- `kind="user"`: a private, END-TO-END ENCRYPTED direct message between two users. The
|
||||
server only ever stores `ciphertext` + `iv` (base64); it never holds the plaintext or
|
||||
the keys, so not even an admin can read it. A conversation is just the set of messages
|
||||
between two users — no separate thread entity; queries group by the {sender, recipient}
|
||||
pair.
|
||||
- `kind="system"`: a server-authored plaintext message (e.g. the welcome, or future admin
|
||||
announcements). `sender_id` is NULL and `body` holds the text. These are non-private
|
||||
boilerplate, readable without any E2EE key setup, and surface as a "Siftlode" conversation.
|
||||
|
||||
`read_at` is stamped when the recipient opens the thread (NULL = unread, feeds the nav
|
||||
badge). The recipient FK cascades (and sender, for user messages) so a GDPR account erasure
|
||||
removes that user's messages outright."""
|
||||
|
||||
__tablename__ = "messages"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
# NULL for system messages (no human sender).
|
||||
sender_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
recipient_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# System messages carry plaintext `body`; user messages carry `ciphertext`+`iv` (base64).
|
||||
body: Mapped[str | None] = mapped_column(Text)
|
||||
ciphertext: Mapped[str | None] = mapped_column(Text)
|
||||
iv: Mapped[str | None] = mapped_column(String(32))
|
||||
read_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), index=True
|
||||
)
|
||||
|
||||
|
||||
class MediaAsset(Base, TimestampMixin):
|
||||
"""A physical downloaded media file — the SHARED cache layer of the download center.
|
||||
|
||||
@@ -830,12 +827,13 @@ class MediaAsset(Base, TimestampMixin):
|
||||
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
||||
shared; per-user edited derivatives (phase 2) are never cached here.
|
||||
|
||||
`rel_path` is relative to `settings.download_root` (the mount), so the physical filename is
|
||||
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
|
||||
system-decided and bound to the source id — never renamed (the user's custom name is display
|
||||
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
||||
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here)
|
||||
plus `last_access_at`/`expires_at` drive retention + LRU eviction in the GC job."""
|
||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
|
||||
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
|
||||
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
|
||||
|
||||
__tablename__ = "media_assets"
|
||||
__table_args__ = (
|
||||
@@ -851,10 +849,10 @@ class MediaAsset(Base, TimestampMixin):
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="pending", server_default="pending", index=True
|
||||
) # "pending" | "ready" | "error"
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
|
||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
||||
# card / watch page / link preview can still show an image. Relative to download_root.
|
||||
# card / watch page / link preview can still show an image. Relative to files_root.
|
||||
poster_path: Mapped[str | None] = mapped_column(Text)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||
@@ -870,8 +868,6 @@ class MediaAsset(Base, TimestampMixin):
|
||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
|
||||
gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||
title: Mapped[str | None] = mapped_column(Text)
|
||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
@@ -881,13 +877,21 @@ class MediaAsset(Base, TimestampMixin):
|
||||
upload_date: Mapped[date | None] = mapped_column(Date)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
||||
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
||||
# reference shown on the Downloads page. Null until the worker extracts it (older/queued rows
|
||||
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
|
||||
# fall back to a URL derived from source_kind+source_ref).
|
||||
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
|
||||
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
|
||||
sha256: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
@validates("rel_path")
|
||||
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
|
||||
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
|
||||
self.sha256 = None
|
||||
return rel_path
|
||||
|
||||
|
||||
class DownloadProfile(Base, TimestampMixin):
|
||||
@@ -911,12 +915,98 @@ class DownloadProfile(Base, TimestampMixin):
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class FileFolder(Base, TimestampMixin):
|
||||
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
|
||||
|
||||
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
|
||||
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
|
||||
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
|
||||
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
|
||||
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
|
||||
|
||||
__tablename__ = "file_folders"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||
Index(
|
||||
"uq_file_folder_own_name",
|
||||
"user_id",
|
||||
func.lower(text("name")),
|
||||
unique=True,
|
||||
postgresql_where=text("name IS NOT NULL"),
|
||||
),
|
||||
# One Hub folder per user.
|
||||
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
|
||||
CheckConstraint(
|
||||
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
||||
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
|
||||
name="ck_file_folder_kind",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
|
||||
# "site:" + a host name (DNS caps one at 253 characters).
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
retention_days: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
|
||||
class ApiToken(Base, TimestampMixin):
|
||||
"""A user's token for a machine to reach their Files library with (`app.apitokens`): sent as
|
||||
`Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its
|
||||
grants name. Only the SHA-256 of the token is kept — it is shown once, when it is made — plus
|
||||
its first characters (`prefix`), so the user can tell their tokens apart."""
|
||||
|
||||
__tablename__ = "api_tokens"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
name: Mapped[str] = mapped_column(String(80))
|
||||
prefix: Mapped[str] = mapped_column(String(16))
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) # NULL = never
|
||||
last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The one notice its user gets before it runs out (`apitokens.warn_expiring`).
|
||||
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
class ApiTokenGrant(Base):
|
||||
"""What one token may do in one folder. A folder with no grant is out of the token's reach."""
|
||||
|
||||
__tablename__ = "api_token_grants"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("token_id", "folder_id", name="uq_api_token_grant"),
|
||||
CheckConstraint(
|
||||
"can_list OR can_read OR can_write OR can_delete", name="ck_api_token_grant_some"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
token_id: Mapped[int] = mapped_column(ForeignKey("api_tokens.id", ondelete="CASCADE"))
|
||||
folder_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
can_list: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
can_read: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
can_write: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
def _ready_on_insert(context) -> datetime | None:
|
||||
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
|
||||
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
|
||||
done = context.get_current_parameters().get("status") == "done"
|
||||
return datetime.now(timezone.utc) if done else None
|
||||
|
||||
|
||||
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||
|
||||
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
|
||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery — the
|
||||
worker is a separate process and can't reach the in-process WS registry). `profile_snapshot`
|
||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
|
||||
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
|
||||
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
|
||||
`display_name` is the user's own name for the file (display + the Content-Disposition on
|
||||
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
|
||||
@@ -929,7 +1019,13 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
|
||||
|
||||
__tablename__ = "download_jobs"
|
||||
__table_args__ = (Index("ix_download_jobs_claim", "status", "queue_pos"),)
|
||||
__table_args__ = (
|
||||
Index("ix_download_jobs_claim", "status", "queue_pos"),
|
||||
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
|
||||
CheckConstraint(
|
||||
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
@@ -938,7 +1034,9 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512))
|
||||
# "download" (default) or "edit" (a per-user trim/crop derivative of another job).
|
||||
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
|
||||
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
|
||||
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
|
||||
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
||||
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
||||
source_job_id: Mapped[int | None] = mapped_column(
|
||||
@@ -947,17 +1045,19 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
source_asset_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("media_assets.id", ondelete="SET NULL")
|
||||
)
|
||||
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}}.
|
||||
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
|
||||
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
|
||||
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
|
||||
edit_spec: Mapped[dict | None] = mapped_column(JSON)
|
||||
profile_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||
)
|
||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
||||
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
||||
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
extra_links: Mapped[list | None] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(
|
||||
@@ -974,6 +1074,27 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
|
||||
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
|
||||
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
|
||||
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
||||
)
|
||||
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
|
||||
# traces its origin through its parent, and that parent may be deleted or expire later — the
|
||||
# file must still know which source folder it may go back to. NULL only for a file placed before
|
||||
# the column existed outside a source folder; `folders.source_key_for` works those out.
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# When the file became done — what the library's "Added" means: a download that failed for days
|
||||
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
|
||||
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
|
||||
ready_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), default=_ready_on_insert
|
||||
)
|
||||
|
||||
|
||||
class DownloadShare(Base, TimestampMixin):
|
||||
@@ -1008,21 +1129,33 @@ class DownloadQuota(Base, UpdatedAtMixin):
|
||||
|
||||
|
||||
class DownloadLink(Base, TimestampMixin):
|
||||
"""A public capability link to one download — the "share by link" surface.
|
||||
"""A public capability link to one download, or to one of its owner's own folders — the "share
|
||||
by link" surface.
|
||||
|
||||
Anyone holding the unguessable `token` can watch the file on the login-free `/watch/{token}`
|
||||
player page (Google-Drive style). Optional per-link controls: `expires_at`, `allow_download`
|
||||
(stream-only vs downloadable), and a `password_hash` (argon2, like a user password). Revoke =
|
||||
delete the row. Distinct from `DownloadShare`, which grants a *registered* user access via the
|
||||
in-app "Shared with me" list."""
|
||||
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
|
||||
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
|
||||
the folder at the time it is opened. The token itself is never stored as written: a link is
|
||||
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
|
||||
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
|
||||
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
|
||||
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
|
||||
grants a *registered* user access via the in-app "Shared with me" list."""
|
||||
|
||||
__tablename__ = "download_links"
|
||||
__table_args__ = (
|
||||
# A link opens exactly one thing: a file or a folder.
|
||||
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
job_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
token: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
password_hash: Mapped[str | None] = mapped_column(Text)
|
||||
@@ -1063,29 +1196,6 @@ class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
||||
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
|
||||
|
||||
class MessageKey(Base, TimestampMixin):
|
||||
"""A user's end-to-end-encryption key material for direct messaging (phase 2).
|
||||
|
||||
One row per user. `public_key` (base64 SPKI, ECDH P-256) is shared with others so they can
|
||||
derive the pairwise conversation key. `wrapped_private_key` is the user's private key
|
||||
encrypted IN THE BROWSER with a key derived (PBKDF2) from a message passphrase the server
|
||||
never sees — so the server stores only an opaque blob it cannot unwrap. `salt`/`wrap_iv`
|
||||
parameterise that derivation; `key_check` is a small ciphertext the client decrypts to
|
||||
verify a passphrase on unlock. The server is purely a key directory + blob store; it can
|
||||
never read any message."""
|
||||
|
||||
__tablename__ = "message_keys"
|
||||
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
public_key: Mapped[str] = mapped_column(Text)
|
||||
wrapped_private_key: Mapped[str] = mapped_column(Text)
|
||||
salt: Mapped[str] = mapped_column(String(64))
|
||||
wrap_iv: Mapped[str] = mapped_column(String(32))
|
||||
key_check: Mapped[str] = mapped_column(Text)
|
||||
|
||||
|
||||
# --- Plex integration (optional) -------------------------------------------------------------
|
||||
# A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is
|
||||
# from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states
|
||||
@@ -1106,7 +1216,7 @@ class PlexLibrary(Base, TimestampMixin, UpdatedAtMixin):
|
||||
kind: Mapped[str] = mapped_column(String(16)) # "movie" | "show"
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# HU-overlay support (E5 S5). `metadata_lang` tags the section's metadata language. An OVERLAY
|
||||
# HU-overlay support. `metadata_lang` tags the section's metadata language. An OVERLAY
|
||||
# library (overlay_of set) is the Hungarian-metadata twin of another section: it is never browsable
|
||||
# on its own — its titles' HU fields are written onto the matched EN rows (paired by `guid`).
|
||||
metadata_lang: Mapped[str] = mapped_column(String(8), default="en", server_default="en")
|
||||
@@ -1135,7 +1245,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# Canonical external id (e.g. "tmdb:12345") — the key that pairs this EN row with its HU-overlay
|
||||
# twin during sync (E5 S5). Indexed for the guid->row overlay match.
|
||||
# twin during sync. Indexed for the guid->row overlay match.
|
||||
guid: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
@@ -1165,7 +1275,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
) # cast+directors, folded into search_vector
|
||||
# Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast so
|
||||
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
||||
imdb_rating: Mapped[float | None] = mapped_column(
|
||||
Float
|
||||
@@ -1175,7 +1285,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
full_cast: Mapped[list | None] = mapped_column(
|
||||
JSONB(none_as_null=True)
|
||||
) # ordered [{name, role, thumb}]
|
||||
# --- TMDB enrichment (E5 S2): see the identical block on plex_items for the full rationale. Additive;
|
||||
# --- TMDB enrichment: see the identical block on plex_items for the full rationale. Additive;
|
||||
# served from the row; NULL until enriched. (`budget`/`revenue` stay NULL for shows — TV has none.) ---
|
||||
tmdb_id: Mapped[int | None] = mapped_column(Integer)
|
||||
tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
@@ -1193,9 +1303,10 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
backdrops: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
||||
# Hungarian-metadata twin fields, written by the overlay sync. NULL = no HU twin matched,
|
||||
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
@@ -1257,7 +1368,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
)
|
||||
kind: Mapped[str] = mapped_column(String(16), index=True) # "movie" | "episode"
|
||||
# Canonical external id (e.g. "tmdb:12345") — pairs a MOVIE row with its HU-overlay twin during
|
||||
# sync (E5 S5); indexed for the guid->row match. Episodes carry it too but it's unused for them.
|
||||
# sync; indexed for the guid->row match. Episodes carry it too but it's unused for them.
|
||||
guid: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
show_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_shows.id", ondelete="CASCADE"), index=True
|
||||
@@ -1284,7 +1395,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# Browser-playability class from ffprobe: "direct" | "remux" | "transcode" (None = unprobed).
|
||||
playable: Mapped[str | None] = mapped_column(String(12), index=True)
|
||||
probed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Transcode HW-capability facet (E7 playback-class filter). NULL for non-transcode items; for a
|
||||
# Transcode HW-capability facet (the playback-class filter). NULL for non-transcode items; for a
|
||||
# `playable="transcode"` item: True = the prod iGPU can hardware-transcode it (codec is HW-decodable
|
||||
# AND the source is 8-bit SDR), False = software libx264 only (AV1/XviD, or a 10-bit/HDR source that
|
||||
# breaks the VAAPI nv12 pipeline — see plex/stream.py). Derived by `plex.sync.transcode_hw_capable`:
|
||||
@@ -1300,8 +1411,6 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# bit-depth/HDR. NULL = never (the backfill's work-list). Only HW-decodable transcode items are
|
||||
# probed (the rest are SW-only by codec and need no bit-depth); see `_probe_media_batch`.
|
||||
media_probed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Intro/credit markers: [{"type": "intro"|"credits", "start_s": int, "end_s": int}, ...]
|
||||
markers: Mapped[list | None] = mapped_column(JSON)
|
||||
added_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
# Filterable / orderable metadata mirrored from the Plex section listing (no extra API calls;
|
||||
# `rating` = Plex audienceRating, which is ~the IMDb score). The arrays back genre/person/studio
|
||||
@@ -1320,7 +1429,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
people_text: Mapped[str | None] = mapped_column(Text)
|
||||
# Set once the movie's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast so
|
||||
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
||||
imdb_rating: Mapped[float | None] = mapped_column(
|
||||
Float
|
||||
@@ -1330,7 +1439,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
full_cast: Mapped[list | None] = mapped_column(
|
||||
JSONB(none_as_null=True)
|
||||
) # ordered [{name, role, thumb}]
|
||||
# --- TMDB enrichment (E5 S2): matched by the canonical `guid` ("tmdb:12345"), filled by the TMDB
|
||||
# --- TMDB enrichment: matched by the canonical `guid` ("tmdb:12345"), filled by the TMDB
|
||||
# backfill job, SERVED FROM THIS ROW (never a live TMDB call). Additive — imdb_rating/rating
|
||||
# above stay Plex-sourced; `tmdb_rating` is the distinct TMDb community score. NULL until enriched;
|
||||
# `tmdb_enriched_at` is the "tried" watermark (stamped on a definitive no-match too, so an
|
||||
@@ -1353,10 +1462,11 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
backdrops: Mapped[list | None] = mapped_column(
|
||||
JSONB(none_as_null=True)
|
||||
) # TMDB backdrop file_paths [str] (crossfade pool, S8)
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
||||
) # TMDB backdrop file_paths [str] (crossfade pool)
|
||||
# Hungarian-metadata twin fields, written by the overlay sync. NULL = no HU twin matched
|
||||
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
@@ -1387,21 +1497,26 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
||||
top few cast. Rows whose name no longer appears in ANY item's cast_names/directors (a library
|
||||
removal orphaned them) are pruned by the maintenance sweep, mirroring how Plex drops orphaned
|
||||
metadata. The filterable cast lives in plex_items/plex_shows.cast_names (self-cleaning with the
|
||||
row); this table is ONLY the headshot cache."""
|
||||
row); this table is the people's own facts: the headshot link, the TMDB link and the TMDB bio
|
||||
(the headshot FILE lives in the image cache, filled by the people_enrich job)."""
|
||||
|
||||
__tablename__ = "plex_people"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), unique=True, index=True)
|
||||
photo_thumb: Mapped[str | None] = mapped_column(Text) # proxied /person-image URL, or None
|
||||
# TMDB person id, linked by name from a title's TMDB credits (E5 S2 lays the foundation; the /person
|
||||
# biography fetch + person-page render land in S6). NULL until a TMDB-enriched title names this person.
|
||||
# TMDB person id, linked by name from a title's TMDB credits (the /person
|
||||
# biography fetch builds on it). NULL until a TMDB-enriched title names this person.
|
||||
tmdb_person_id: Mapped[int | None] = mapped_column(Integer)
|
||||
# --- TMDB person enrichment (E5 S6): fetched from /person/{tmdb_person_id} the first time the person
|
||||
# page is opened, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||
# --- TMDB person enrichment: fetched from /person/{tmdb_person_id} by the people_enrich job,
|
||||
# or by the first person-page open the job hasn't reached, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||
# no-op fetch so it isn't retried every open). `known_for` = top combined-credits [{tmdb_id,
|
||||
# media_type, title}], resolved to library rows by guid at serve time (like recommendations). ---
|
||||
biography: Mapped[str | None] = mapped_column(Text)
|
||||
# TMDB's Hungarian biography, from the same request's `translations`; a Hungarian reader is
|
||||
# served it, and the instance-language `biography` when TMDB has none. Birthplace, aliases and
|
||||
# keywords have no Hungarian on TMDB, so they stay in the instance language.
|
||||
biography_hu: Mapped[str | None] = mapped_column(Text)
|
||||
birthday: Mapped[str | None] = mapped_column(String(10)) # "YYYY-MM-DD", served as-is
|
||||
deathday: Mapped[str | None] = mapped_column(String(10))
|
||||
place_of_birth: Mapped[str | None] = mapped_column(Text)
|
||||
@@ -1439,7 +1554,7 @@ class PlexCollection(Base, TimestampMixin, UpdatedAtMixin):
|
||||
class PlexState(Base, UpdatedAtMixin):
|
||||
"""Per-user watch state for a Plex item — mirrors VideoState. Lives in Siftlode's own DB so
|
||||
it works for users WITHOUT a Plex account (the whole point of the access-layer design).
|
||||
`synced_to_plex` marks whether this state was pushed back to a linked Plex account (P5)."""
|
||||
`synced_to_plex` marks whether this state was pushed back to a linked Plex account."""
|
||||
|
||||
__tablename__ = "plex_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "item_id", name="uq_plex_user_item"),)
|
||||
@@ -1518,7 +1633,7 @@ class PlexPlaylistItem(Base):
|
||||
|
||||
|
||||
class ComfortPin(Base, TimestampMixin):
|
||||
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X" (E24).
|
||||
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X".
|
||||
|
||||
Deliberately NOT a boolean on `VideoState`/`PlexState`, and NOT a polymorphic `(kind, ref)` string
|
||||
pair. A flag cannot ORDER a shelf and cannot hold a channel or a show — and comfort media is
|
||||
|
||||
@@ -10,4 +10,4 @@ from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
|
||||
CACHE_DIR = Path(settings.download_root) / ".plex-img-cache"
|
||||
CACHE_DIR = Path(settings.files_root) / ".plex-img-cache"
|
||||
|
||||
+49
-15
@@ -3,9 +3,24 @@ playlists, and detail views render. Extracted from routes/plex/browse.py so the
|
||||
lives beside the other Plex read-model modules (detail.py, query.py) rather than inside the
|
||||
router. Pure functions — no DB, no request state."""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
from app.models import PlexItem, PlexShow, PlexState
|
||||
|
||||
|
||||
class ShowState(NamedTuple):
|
||||
"""A show's watch state rolled up from its episodes, as its card carries it. `resumable`: an unwatched
|
||||
episode has a resume point, the one the card's Continue resumes and its Restart replays from 0. A
|
||||
show part watched with its next episode untouched has none: Continue and Restart would both play
|
||||
that episode from its start, so the card offers Play alone."""
|
||||
|
||||
status: str
|
||||
resumable: bool
|
||||
|
||||
|
||||
SHOW_NEW = ShowState("new", False)
|
||||
|
||||
|
||||
# A card's wide backdrop, for a rail's landscape tile or a crossfading hero — but ONLY when the item
|
||||
# actually has art. `thumb` is always emitted (its endpoint placeholders a missing poster), but an
|
||||
# `art` URL for an item with no `art_key` would resolve to a broken/blank image on every card that
|
||||
@@ -15,20 +30,35 @@ def _art_url(rating_key: str, art_key: str | None) -> str | None:
|
||||
|
||||
|
||||
# Card hover-overview: a short synopsis for the reveal panel, trimmed so a strip/related row of cards
|
||||
# stays a few hundred bytes each rather than carrying full synopses. (These non-grid card paths serve
|
||||
# the EN summary/rating; the main grid coalesces the HU twin in browse.unified_library.)
|
||||
# stays a few hundred bytes each rather than carrying full synopses.
|
||||
def _card_summary(summary: str | None) -> str | None:
|
||||
return (summary or "")[:300] or None
|
||||
|
||||
|
||||
def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
||||
def norm_lang(lang: str | None) -> str:
|
||||
"""The overlay has an EN base + an optional HU twin; anything other than `hu` serves EN."""
|
||||
return "hu" if lang == "hu" else "en"
|
||||
|
||||
|
||||
# Every card speaks the reader's language: `lang="hu"` serves the HU overlay twin's title and
|
||||
# summary, falling back to EN where a row has none — the same rule as browse.unified_library's
|
||||
# `title_col`/`summary_col`. These rails once served English to a Hungarian reader.
|
||||
def display_title(row: PlexItem | PlexShow, lang: str) -> str:
|
||||
return (row.title_hu or row.title) if lang == "hu" else row.title
|
||||
|
||||
|
||||
def display_summary(row: PlexItem | PlexShow, lang: str) -> str | None:
|
||||
return (row.summary_hu or row.summary) if lang == "hu" else row.summary
|
||||
|
||||
|
||||
def movie_card(it: PlexItem, st: PlexState | None, *, lang: str) -> dict:
|
||||
return {
|
||||
"id": it.rating_key,
|
||||
"type": "movie",
|
||||
"title": it.title,
|
||||
"title": display_title(it, lang),
|
||||
"year": it.year,
|
||||
"rating": it.rating,
|
||||
"summary": _card_summary(it.summary),
|
||||
"summary": _card_summary(display_summary(it, lang)),
|
||||
"duration_seconds": it.duration_s,
|
||||
"thumb": f"/api/plex/image/{it.rating_key}",
|
||||
"art": _art_url(it.rating_key, it.art_key),
|
||||
@@ -39,28 +69,29 @@ def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def show_card(sh: PlexShow, status: str = "new") -> dict:
|
||||
def show_card(sh: PlexShow, state: ShowState = SHOW_NEW, *, lang: str) -> dict:
|
||||
return {
|
||||
"id": sh.rating_key,
|
||||
"type": "show",
|
||||
"title": sh.title,
|
||||
"title": display_title(sh, lang),
|
||||
"year": sh.year,
|
||||
"rating": sh.rating,
|
||||
"summary": _card_summary(sh.summary),
|
||||
"summary": _card_summary(display_summary(sh, lang)),
|
||||
"thumb": f"/api/plex/image/{sh.rating_key}",
|
||||
"art": _art_url(sh.rating_key, sh.art_key),
|
||||
"season_count": sh.child_count,
|
||||
# Aggregate watch-state across the show's episodes (new | in_progress | watched) → grid badge.
|
||||
"status": status,
|
||||
"status": state.status,
|
||||
"resumable": state.resumable,
|
||||
}
|
||||
|
||||
|
||||
def episode_card(e: PlexItem, st: PlexState | None) -> dict:
|
||||
def episode_card(e: PlexItem, st: PlexState | None, *, lang: str) -> dict:
|
||||
return {
|
||||
"id": e.rating_key,
|
||||
"type": "episode",
|
||||
"title": e.title,
|
||||
"summary": e.summary,
|
||||
"title": display_title(e, lang),
|
||||
"summary": display_summary(e, lang),
|
||||
"season_number": e.season_number,
|
||||
"episode_number": e.episode_number,
|
||||
"duration_seconds": e.duration_s,
|
||||
@@ -77,12 +108,15 @@ def leaf_card(
|
||||
st: PlexState | None,
|
||||
show_title: str | None = None,
|
||||
show_rk: str | None = None,
|
||||
*,
|
||||
lang: str,
|
||||
) -> dict:
|
||||
"""A playable-leaf card (movie or episode). Used by playlists, which can mix both kinds.
|
||||
`show_rk` (the show's rating_key) lets the client group a playlist's episodes by show/season."""
|
||||
`show_rk` (the show's rating_key) lets the client group a playlist's episodes by show/season.
|
||||
`show_title` is the caller's, already in `lang` (see `display_title`)."""
|
||||
if it.kind == "episode":
|
||||
card = episode_card(it, st)
|
||||
card = episode_card(it, st, lang=lang)
|
||||
card["show_title"] = show_title
|
||||
card["show_id"] = show_rk
|
||||
return card
|
||||
return movie_card(it, st)
|
||||
return movie_card(it, st, lang=lang)
|
||||
|
||||
+28
-15
@@ -2,7 +2,7 @@
|
||||
|
||||
Server-side ONLY: uses the admin ``X-Plex-Token`` from sysconfig. Playback never goes through
|
||||
this client — Siftlode plays the LOCAL physical file (see app.plex.paths). Plex is used purely for
|
||||
metadata, image proxying, and (P5) optional watch-state write-back. Requests JSON via the
|
||||
metadata, image proxying, and optional watch-state write-back. Requests JSON via the
|
||||
``Accept: application/json`` header (Plex serves XML by default).
|
||||
"""
|
||||
|
||||
@@ -65,7 +65,7 @@ class PlexClient:
|
||||
except ValueError as e:
|
||||
raise PlexError("Plex returned a non-JSON response") from e
|
||||
|
||||
# --- Connectivity + sections (P0) ---
|
||||
# --- Connectivity + sections ---
|
||||
def server_info(self) -> dict:
|
||||
"""Root identity (friendlyName / version) — used by the admin 'Test connection'."""
|
||||
return self._get("/")
|
||||
@@ -75,25 +75,29 @@ class PlexClient:
|
||||
mc = self._get("/library/sections")
|
||||
return mc.get("Directory", []) or []
|
||||
|
||||
# --- Catalog mirror (P1) ---
|
||||
# --- Catalog mirror ---
|
||||
def section_items(
|
||||
self, key: str, item_type: int, start: int = 0, size: int = 200
|
||||
) -> tuple[list[dict], int]:
|
||||
) -> tuple[list[dict], int | None]:
|
||||
"""A page of a section's top-level items. type 1 = movie, 2 = show. Returns
|
||||
(items, total_size)."""
|
||||
(items, total_size); the total is None when the answer carries none, so a container that
|
||||
says "0" (a genuinely empty section) stays distinguishable from one that says nothing."""
|
||||
mc = self._get(
|
||||
f"/library/sections/{key}/all",
|
||||
params={
|
||||
"type": item_type,
|
||||
# includeGuids=1 carries each item's Guid array (tmdb/imdb/tvdb ids) in the cheap
|
||||
# listing, so the sync can capture the canonical guid without a per-item rich fetch —
|
||||
# the pairing key for the HU overlay (E5 S5).
|
||||
# the pairing key for the HU overlay.
|
||||
"includeGuids": 1,
|
||||
"X-Plex-Container-Start": start,
|
||||
"X-Plex-Container-Size": size,
|
||||
},
|
||||
)
|
||||
return mc.get("Metadata", []) or [], int(mc.get("totalSize", mc.get("size", 0)) or 0)
|
||||
# `totalSize` only: `size` is THIS page's count, and read as the total it made a single
|
||||
# page look like the whole section — "complete" to a prune or a reconcile.
|
||||
total = mc.get("totalSize")
|
||||
return mc.get("Metadata", []) or [], None if total is None else int(total)
|
||||
|
||||
def children(self, rating_key: str) -> list[dict]:
|
||||
"""Direct children (show → seasons, season → episodes)."""
|
||||
@@ -133,12 +137,17 @@ class PlexClient:
|
||||
mc = self._get(f"/library/collections/{rating_key}/children")
|
||||
return mc.get("Metadata", []) or []
|
||||
|
||||
# --- Collection editing (P2 — write-back to Plex; all shared/global, admin-gated in the routes) ---
|
||||
# --- Collection editing (write-back to Plex; all shared/global, admin-gated in the routes) ---
|
||||
@property
|
||||
def machine_id(self) -> str:
|
||||
"""Server machineIdentifier — needed in the `uri` of collection add/create calls. Cached."""
|
||||
"""Server machineIdentifier — needed in the `uri` of collection add/create calls. Cached.
|
||||
Without one the uri would read `server:///…`, a malformed write Plex answers in its own way;
|
||||
it is refused here, by name."""
|
||||
if not getattr(self, "_machine_id", None):
|
||||
self._machine_id = self._get("/identity").get("machineIdentifier") or ""
|
||||
mid = self._get("/identity").get("machineIdentifier")
|
||||
if not mid:
|
||||
raise PlexError("Plex did not report its machine identifier")
|
||||
self._machine_id = mid
|
||||
return self._machine_id
|
||||
|
||||
def _write(self, method: str, path: str) -> dict:
|
||||
@@ -158,12 +167,16 @@ class PlexClient:
|
||||
keys = ",".join(str(k) for k in rating_keys)
|
||||
return f"server://{self.machine_id}/com.plexapp.plugins.library/library/metadata/{keys}"
|
||||
|
||||
def create_collection(self, section_key: str, title: str, item_rating_key: str) -> dict:
|
||||
"""Create a (dumb) collection seeded with one item; returns its Metadata (incl. ratingKey)."""
|
||||
def create_collection(
|
||||
self, section_key: str, title: str, item_rating_key: str, item_type: int = 1
|
||||
) -> dict:
|
||||
"""Create a (dumb) collection seeded with one item; returns its Metadata (incl. ratingKey).
|
||||
`item_type` is the Plex metadata type of its members — 1 movie, 2 show. Measured: a collection
|
||||
made as type 1 in a TV section came out EMPTY with a show seed, and refused the show after."""
|
||||
uri = quote(self._metadata_uri(item_rating_key), safe="")
|
||||
mc = self._write(
|
||||
"POST",
|
||||
f"/library/collections?type=1&smart=0§ionId={section_key}"
|
||||
f"/library/collections?type={item_type}&smart=0§ionId={section_key}"
|
||||
f"&title={quote(title, safe='')}&uri={uri}",
|
||||
)
|
||||
items = mc.get("Metadata", []) or []
|
||||
@@ -197,7 +210,7 @@ class PlexClient:
|
||||
items = mc.get("Metadata", []) or []
|
||||
return items[0] if items else None
|
||||
|
||||
# --- Watch-state write-back (P5 Phase B — Siftlode→Plex push; acts on the token's account) ---
|
||||
# --- Watch-state write-back (Siftlode→Plex push; acts on the token's account) ---
|
||||
_LIBRARY_IDENTIFIER = "com.plexapp.plugins.library"
|
||||
|
||||
def scrobble(self, rating_key: str) -> None:
|
||||
@@ -234,7 +247,7 @@ class PlexClient:
|
||||
f"&state={state}&time={int(time_ms)}&duration={int(duration_ms)}",
|
||||
)
|
||||
|
||||
# --- Watch-state read for incremental sync (P5 Phase C) ---
|
||||
# --- Watch-state read for incremental sync ---
|
||||
def accounts(self) -> list[dict]:
|
||||
"""Server accounts (the owner + any managed/home users): id, name. Used to resolve the
|
||||
owner's accountID so the shared watch-history feed can be filtered to just their views."""
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Parse the "info page" rich metadata out of a full Plex metadata dict, and persist the subset that
|
||||
the detail endpoints serve from the DB row (E5 S5b).
|
||||
the detail endpoints serve from the DB row.
|
||||
|
||||
Shared by the live info-page path (routes.plex.browse) and the background rich re-sync (plex.sync) so
|
||||
the two never diverge — the same one-parser rule that keeps `people.py`'s cast handling consistent.
|
||||
@@ -7,6 +7,7 @@ the two never diverge — the same one-parser rule that keeps `people.py`'s cast
|
||||
(genres/directors/studio/content_rating already land from the cheap listing at `_apply_facet_fields`)."""
|
||||
|
||||
from app.plex import people
|
||||
from app.plex.plaintext import plain_text
|
||||
|
||||
|
||||
def rich_detail(meta: dict) -> dict:
|
||||
@@ -42,7 +43,7 @@ def rich_detail(meta: dict) -> dict:
|
||||
"genres": [g.get("tag") for g in (meta.get("Genre") or []) if g.get("tag")][:8],
|
||||
"directors": [d.get("tag") for d in (meta.get("Director") or []) if d.get("tag")][:4],
|
||||
"studio": meta.get("studio"),
|
||||
"tagline": meta.get("tagline"),
|
||||
"tagline": plain_text(meta.get("tagline")),
|
||||
"cast": people.cast_from_meta(meta),
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""One Hungarian word per Plex genre.
|
||||
|
||||
Plex's Hungarian agent returns a MIXED vocabulary: some genres come back translated ("Dráma",
|
||||
"Vígjáték"), others stay English ("Action", "Adventure", "Biography"), and a title with no HU twin has
|
||||
only its English genres. The HU genre list therefore offered "Drama" beside "Dráma" and a dozen English
|
||||
words. This map sends every genre word we have met — English (Plex EN agent, TMDB, IMDb) or Plex's
|
||||
Hungarian — to the ONE word a Hungarian reader sees. The overlay sync writes it into `genres_hu`, and a
|
||||
genre filter is read through it too, so a saved filter still holding "Action" keeps matching "Akció".
|
||||
|
||||
A word not in the map passes through unchanged: an unknown genre stays visible (in whatever language
|
||||
Plex gave it) rather than disappearing.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
# Keys are matched case-insensitively (see `_BY_KEY`). The Hungarian values double as keys mapping to
|
||||
# themselves, so running a list through twice changes nothing.
|
||||
_HU: dict[str, str] = {
|
||||
"Action": "Akció",
|
||||
"Action & Adventure": "Akció és kaland",
|
||||
"Adventure": "Kaland",
|
||||
"Animation": "Animációs",
|
||||
"Anime": "Anime",
|
||||
"Biography": "Életrajzi",
|
||||
"Children": "Gyerek",
|
||||
"Comedy": "Vígjáték",
|
||||
"Crime": "Bűnügyi",
|
||||
"Documentary": "Dokumentum",
|
||||
"Drama": "Dráma",
|
||||
"Family": "Családi",
|
||||
"Fantasy": "Fantasy",
|
||||
"Film-Noir": "Film noir",
|
||||
"Food": "Gasztronómia",
|
||||
"Game Show": "Vetélkedő",
|
||||
"Game-Show": "Vetélkedő",
|
||||
"History": "Történelmi",
|
||||
"Home and Garden": "Otthon és kert",
|
||||
"Horror": "Horror",
|
||||
"Indie": "Független",
|
||||
"Kids": "Gyerek",
|
||||
"Martial Arts": "Harcművészet",
|
||||
"Mini-Series": "Minisorozat",
|
||||
"Music": "Zenei",
|
||||
"Musical": "Musical",
|
||||
"Mystery": "Rejtély",
|
||||
"News": "Hírek",
|
||||
"Reality": "Valóságshow",
|
||||
"Reality-TV": "Valóságshow",
|
||||
"Romance": "Romantikus",
|
||||
"Sci-Fi": "Sci-Fi",
|
||||
"Sci-Fi & Fantasy": "Sci-Fi és fantasy",
|
||||
"Science Fiction": "Sci-Fi",
|
||||
"Short": "Rövidfilm",
|
||||
"Soap": "Szappanopera",
|
||||
"Sport": "Sport",
|
||||
"Suspense": "Feszültség",
|
||||
"Talk": "Talkshow",
|
||||
"Talk Show": "Talkshow",
|
||||
"Thriller": "Thriller",
|
||||
"Travel": "Utazás",
|
||||
"TV Movie": "TV film",
|
||||
"War": "Háborús",
|
||||
"War & Politics": "Háború és politika",
|
||||
"Western": "Western",
|
||||
# Plex's own Hungarian words, pinned to themselves (and to the spelling above where they differ).
|
||||
"Akció": "Akció",
|
||||
"Animációs": "Animációs",
|
||||
"Bűnügyi": "Bűnügyi",
|
||||
"Családi": "Családi",
|
||||
"Dokumentum": "Dokumentum",
|
||||
"Dokumentumfilm": "Dokumentum",
|
||||
"Dráma": "Dráma",
|
||||
"Életrajzi": "Életrajzi",
|
||||
"Háborús": "Háborús",
|
||||
"Kaland": "Kaland",
|
||||
"Rejtély": "Rejtély",
|
||||
"Romantikus": "Romantikus",
|
||||
"Történelmi": "Történelmi",
|
||||
"TV film": "TV film",
|
||||
"Vígjáték": "Vígjáték",
|
||||
"Zene": "Zenei",
|
||||
"Zenei": "Zenei",
|
||||
}
|
||||
_BY_KEY = {k.casefold(): v for k, v in _HU.items()}
|
||||
|
||||
|
||||
def hu_genre(word: str) -> str:
|
||||
"""The Hungarian word for one genre; an unknown word comes back unchanged."""
|
||||
return _BY_KEY.get(word.strip().casefold(), word)
|
||||
|
||||
|
||||
def hu_genres(words: Iterable[str] | None) -> list[str] | None:
|
||||
"""A genre list in Hungarian, in its first-seen order, each word once ("Drama" + "Dráma" → one
|
||||
"Dráma"). None or empty stays None, like the listing's own `_tags`."""
|
||||
out: list[str] = []
|
||||
for w in words or ():
|
||||
hu = hu_genre(w)
|
||||
if hu not in out:
|
||||
out.append(hu)
|
||||
return out or None
|
||||
@@ -8,6 +8,9 @@ few hundred MB at most for a big library — acceptable, matching the download d
|
||||
The cache directory (``cache.CACHE_DIR``) is shared with the subtitle cache (see ``subtitles.py``).
|
||||
"""
|
||||
|
||||
import time
|
||||
from datetime import timedelta
|
||||
|
||||
from fastapi import Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -20,9 +23,23 @@ log = logs.get_logger(__name__)
|
||||
|
||||
_EXT_CT = [("jpg", "image/jpeg"), ("png", "image/png"), ("webp", "image/webp")]
|
||||
_CT_EXT = {ct: ext for ext, ct in _EXT_CT}
|
||||
# Target box per variant — Plex resizes to these server-side. Posters fill ≤176px cells (grid/info/
|
||||
# strips), so 400px covers 2× DPR; the faint backdrop art only needs ~1280px. Keeps images ~tens of KB.
|
||||
SIZES = {"thumb": (400, 600), "art": (1280, 720)}
|
||||
# Target box per variant — Plex resizes to these server-side. The full poster (400px) covers a detail
|
||||
# page's ≤224px poster at 2× DPR, the full art (1280px) a hero backdrop. A card is much smaller: a grid
|
||||
# poster is ~144px and a shelf's landscape art ~212px, so the `_sm` pair serves those at 1.33× DPR
|
||||
# (cards used to fetch the 400 poster, 42 KB, and the 1280 art for a 212px picture). The browser
|
||||
# picks between a pair itself (`srcset`, built by the frontend's `lib/plexImage` cardImage, whose
|
||||
# widths plexImage.contract.test holds to these).
|
||||
SIZES = {
|
||||
"thumb": (400, 600),
|
||||
"thumb_sm": (240, 360),
|
||||
"art": (1280, 720),
|
||||
"art_sm": (480, 270),
|
||||
}
|
||||
|
||||
|
||||
def picture(variant: str) -> str:
|
||||
"""Which stored picture a size variant is cut from: the poster (`thumb`) or the backdrop (`art`)."""
|
||||
return "art" if variant.startswith("art") else "thumb"
|
||||
|
||||
|
||||
def key_for(db: Session, rating_key: str, variant: str) -> str | None:
|
||||
@@ -56,6 +73,49 @@ def read(key: str) -> tuple[bytes, str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def has(key: str) -> bool:
|
||||
"""A cached, non-empty file exists for `key` (a prefetch's cheap check — no read)."""
|
||||
for ext, _ in _EXT_CT:
|
||||
try:
|
||||
if (CACHE_DIR / f"{key}.{ext}").stat().st_size > 0:
|
||||
return True
|
||||
# silent-ok: a miss, as in read().
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def drop(key: str) -> None:
|
||||
"""Remove `key`'s file, whatever its extension (a superseded variant). Best-effort."""
|
||||
for ext, _ in _EXT_CT:
|
||||
try:
|
||||
(CACHE_DIR / f"{key}.{ext}").unlink(missing_ok=True)
|
||||
# silent-ok: a leftover original only costs disk; the small twin is already written.
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def mark_miss(key: str) -> None:
|
||||
"""Remember that `key`'s source could not be fetched (an empty `.miss` file; its mtime is when), so
|
||||
a prefetch can leave a dead source alone for a while. Best-effort."""
|
||||
try:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(CACHE_DIR / f"{key}.miss").touch()
|
||||
# silent-ok: without the mark the dead source is only asked for again on the next run.
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def missed_within(key: str, age: timedelta) -> bool:
|
||||
"""`key` was marked as a miss less than `age` ago."""
|
||||
try:
|
||||
mtime = (CACHE_DIR / f"{key}.miss").stat().st_mtime
|
||||
# silent-ok: no mark (the usual case).
|
||||
except OSError:
|
||||
return False
|
||||
return time.time() - mtime < age.total_seconds()
|
||||
|
||||
|
||||
def write(key: str, data: bytes, ct: str) -> None:
|
||||
ext = _CT_EXT.get((ct or "").split(";")[0].strip(), "jpg")
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
"""The HU overlay: a Hungarian Plex section written onto its English twin's rows, split out of
|
||||
`plex/sync.py`, which runs it once per paired section during the full sync.
|
||||
|
||||
The pairing is admin-configured (`plex_overlay_map`, ``huKey:enKey,...``). An overlay section creates
|
||||
no browsable rows of its own; it only fills the `*_hu` columns of the EN rows it matches by guid.
|
||||
`plex/sync.py` and this module import each other as MODULES (attribute access at call time), which is
|
||||
what lets either be imported first.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexItem, PlexLibrary, PlexShow
|
||||
from app.plex import sync as plex_sync
|
||||
from app.plex.client import PlexClient
|
||||
from app.plex.genres import hu_genres
|
||||
from app.plex.plaintext import plain_text
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def overlay_map(db: Session) -> dict[str, str]:
|
||||
"""The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
|
||||
('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
|
||||
Malformed pairs are skipped."""
|
||||
raw = sysconfig.get_str(db, "plex_overlay_map")
|
||||
out: dict[str, str] = {}
|
||||
for pair in raw.split(","):
|
||||
hu, sep, en = pair.partition(":")
|
||||
hu, en = hu.strip(), en.strip()
|
||||
if sep and hu and en:
|
||||
out[hu] = en
|
||||
return out
|
||||
|
||||
|
||||
def sync_overlay(
|
||||
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
||||
) -> None:
|
||||
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/tagline/genres
|
||||
differ by language; the language-neutral people_text feeds both search vectors. A show section's
|
||||
episodes are overlaid too: the HU section lists every episode with its Hungarian title and
|
||||
summary, and the episode rows live in the EN show library.
|
||||
|
||||
This is a full RECONCILE: after a COMPLETE HU listing every guid-bearing EN row's HU fields are
|
||||
cleared, then the matches are repopulated — so an EN row whose HU twin vanished (or was re-guid'd)
|
||||
falls back to EN instead of keeping stale Hungarian metadata. A listing cut short only adds. HU
|
||||
items with no EN match are skipped."""
|
||||
if lib.kind == "movie":
|
||||
passes = [
|
||||
(
|
||||
plex_sync._TYPE_MOVIE,
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.guid.is_not(None),
|
||||
),
|
||||
)
|
||||
]
|
||||
else:
|
||||
passes = [
|
||||
(
|
||||
plex_sync._TYPE_SHOW,
|
||||
select(PlexShow).where(
|
||||
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
|
||||
),
|
||||
),
|
||||
(
|
||||
plex_sync._TYPE_EPISODE,
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "episode",
|
||||
PlexItem.guid.is_not(None),
|
||||
),
|
||||
),
|
||||
]
|
||||
for item_type, rows_q in passes:
|
||||
_overlay_pass(db, plex, lib, item_type, db.execute(rows_q).scalars(), stats)
|
||||
db.flush()
|
||||
|
||||
|
||||
def _overlay_pass(
|
||||
db: Session,
|
||||
plex: PlexClient,
|
||||
lib: PlexLibrary,
|
||||
item_type: int,
|
||||
rows: Iterable[PlexItem] | Iterable[PlexShow],
|
||||
stats: dict,
|
||||
) -> None:
|
||||
"""One kind of the overlay (movies, shows or episodes) — see `sync_overlay`."""
|
||||
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
||||
by_guid: dict[str, list[PlexItem | PlexShow]] = {}
|
||||
for r in rows:
|
||||
if r.guid: # the passes select guid-bearing rows only
|
||||
by_guid.setdefault(r.guid, []).append(r)
|
||||
# The HU listing is read first and applied after, so the clear can wait for its verdict.
|
||||
hu_by_guid, complete = _read_hu_listing(plex, lib, item_type, by_guid)
|
||||
episodes = item_type == plex_sync._TYPE_EPISODE
|
||||
if complete:
|
||||
_clear_hu(by_guid, episodes)
|
||||
else:
|
||||
stats["overlay_incomplete"] = stats.get("overlay_incomplete", 0) + 1
|
||||
log.warning(
|
||||
"the HU listing ended before the server's total; existing HU text is kept",
|
||||
extra=logs.event("plex.sync.overlay_incomplete", library=lib.plex_key),
|
||||
)
|
||||
for g, meta in hu_by_guid.items():
|
||||
targets = by_guid[g]
|
||||
title_hu = (meta.get("title") or "").strip() or None
|
||||
summary_hu = plain_text(meta.get("summary"))
|
||||
tagline_hu = plain_text(meta.get("tagline"))
|
||||
genres_hu = hu_genres(plex_sync._tags(meta, "Genre"))
|
||||
for row in targets:
|
||||
row.title_hu = title_hu
|
||||
row.summary_hu = summary_hu
|
||||
if not episodes:
|
||||
row.tagline_hu = tagline_hu
|
||||
row.genres_hu = genres_hu or row.genres_hu
|
||||
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
||||
|
||||
|
||||
def _read_hu_listing(
|
||||
plex: PlexClient, lib: PlexLibrary, item_type: int, by_guid: dict
|
||||
) -> tuple[dict[str, dict], bool]:
|
||||
"""The HU section's entries for the guids the EN side carries, and whether the walk reached the
|
||||
server's total. The first HU entry for a guid wins; a dup HU item can't overwrite or double-count."""
|
||||
progress: dict = {}
|
||||
hu_by_guid: dict[str, dict] = {}
|
||||
for meta in plex_sync._paginate(plex, lib.plex_key, item_type, progress):
|
||||
g = plex_sync._canonical_guid(meta)
|
||||
if g and g in by_guid and g not in hu_by_guid:
|
||||
hu_by_guid[g] = meta
|
||||
return hu_by_guid, bool(progress.get("complete"))
|
||||
|
||||
|
||||
def _clear_hu(by_guid: dict, episodes: bool) -> None:
|
||||
"""Reconcile step: clear HU fields on all guid-bearing rows; the matches are re-set after. Only
|
||||
rows that actually held HU data become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is
|
||||
cheap. The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking
|
||||
one, from its own English list — otherwise the HU genre list mixed "Drama" (no twin) with "Dráma".
|
||||
Only after a COMPLETE listing: one a Plex hiccup cut short would clear the HU text of every title
|
||||
it failed to reach, until the next full sync (the watch reconcile's twin)."""
|
||||
for group in by_guid.values():
|
||||
for row in group:
|
||||
row.title_hu = row.summary_hu = None
|
||||
if not episodes:
|
||||
row.tagline_hu = None
|
||||
row.genres_hu = hu_genres(row.genres)
|
||||
+176
-10
@@ -3,13 +3,23 @@ or show (cast_names + the people_text search blob + the plex_people headshot cac
|
||||
orphaned people. Used by BOTH the info-page lazy enrichment (routes.plex) and the background rich
|
||||
re-sync (plex.sync), so the two paths never diverge."""
|
||||
|
||||
from urllib.parse import quote
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import timedelta
|
||||
from typing import Literal
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexPerson
|
||||
from app.plex import imgcache
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Cast/crew photos come from Plex's PUBLIC metadata CDN (no token). We proxy them (host-whitelisted)
|
||||
# so the user's browser makes no third-party request (privacy + self-host CSP friendliness).
|
||||
@@ -19,9 +29,7 @@ PERSON_IMG_HOST = "https://metadata-static.plex.tv/"
|
||||
def person_thumb_url(thumb: str | None) -> str | None:
|
||||
"""Proxied /person-image URL for a Plex CDN headshot, or None when it's missing / off-CDN."""
|
||||
t = str(thumb or "")
|
||||
return (
|
||||
f"/api/plex/person-image?u={quote(t, safe='')}" if t.startswith(PERSON_IMG_HOST) else None
|
||||
)
|
||||
return f"/api/plex/person-image?u={quote(t, safe='')}" if is_portrait_url(t) else None
|
||||
|
||||
|
||||
def cast_from_meta(meta: dict, limit: int = 24) -> list[dict]:
|
||||
@@ -89,14 +97,23 @@ def enrich_row_people(db: Session, row, cast: list[dict]) -> bool:
|
||||
return dirty
|
||||
|
||||
|
||||
# Every credit reference: one person name per (movie/show, credit column). The ONE list of credit
|
||||
# columns, so the orphan prune and the enrichment order never disagree about who is credited.
|
||||
_CREDIT_COLUMNS = [
|
||||
(table, col) for table in ("plex_items", "plex_shows") for col in ("cast_names", "directors")
|
||||
]
|
||||
|
||||
|
||||
def _credit_refs_sql(union: Literal["UNION", "UNION ALL"]) -> str:
|
||||
return f" {union} ".join(
|
||||
f"SELECT jsonb_array_elements_text({col}) FROM {table} WHERE jsonb_typeof({col}) = 'array'"
|
||||
for table, col in _CREDIT_COLUMNS
|
||||
)
|
||||
|
||||
|
||||
# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside
|
||||
# this set are orphans left by a library removal.
|
||||
_REFERENCED_PEOPLE_SQL = """
|
||||
SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array'
|
||||
"""
|
||||
_REFERENCED_PEOPLE_SQL = _credit_refs_sql("UNION")
|
||||
|
||||
|
||||
def prune_orphan_people(db: Session) -> int:
|
||||
@@ -113,3 +130,152 @@ def prune_orphan_people(db: Session) -> int:
|
||||
)
|
||||
db.commit()
|
||||
return res.rowcount or 0
|
||||
|
||||
|
||||
# How often each person is credited across movies, shows and episodes (cast OR director) — the order the
|
||||
# people_enrich job works in, so the faces met most often are local first.
|
||||
_CREDIT_COUNTS_SQL = (
|
||||
f"SELECT ref, count(*) AS n FROM ({_credit_refs_sql('UNION ALL')}) AS refs(ref) GROUP BY ref"
|
||||
)
|
||||
_NEEDS = {
|
||||
# linked to TMDB, bio never fetched
|
||||
"bio": "p.tmdb_person_id IS NOT NULL AND p.tmdb_enriched_at IS NULL",
|
||||
# has a Plex headshot (whether it is cached is the disk's answer, not the row's)
|
||||
"portrait": "p.photo_thumb IS NOT NULL",
|
||||
}
|
||||
|
||||
|
||||
def people_by_credits(
|
||||
db: Session, need: Literal["bio", "portrait"], limit: int | None
|
||||
) -> list[int]:
|
||||
"""plex_people ids that `need` something, the most-credited first (ties by id)."""
|
||||
rows = db.execute(
|
||||
text(
|
||||
f"SELECT p.id FROM plex_people p LEFT JOIN ({_CREDIT_COUNTS_SQL}) c ON c.ref = p.name "
|
||||
f"WHERE {_NEEDS[need]} ORDER BY coalesce(c.n, 0) DESC, p.id"
|
||||
+ (" LIMIT :limit" if limit is not None else "")
|
||||
),
|
||||
{"limit": limit} if limit is not None else {},
|
||||
)
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
# --- Portraits ---------------------------------------------------------------------------
|
||||
# A Plex CDN headshot is a full-size original (~170 KB on average, measured). Plex's photo transcoder
|
||||
# fits it into this box (~13 KB), which covers the largest place a portrait is drawn (the person page's
|
||||
# 6rem avatar) at 2x density and a raised text size. The box is part of the cache key, like a poster's:
|
||||
# a new box re-caches instead of serving a stale size. Only a resized image is ever written under it —
|
||||
# an original fetched without Plex keeps its own key, and goes once its small twin lands.
|
||||
PORTRAIT_BOX = 320
|
||||
# A headshot Plex answered "not found" for (a dead CDN file) is not asked for again this long, so a
|
||||
# few dead faces never hold up the rest of the prefetch.
|
||||
PORTRAIT_RETRY = timedelta(days=7)
|
||||
# The one shape a Plex CDN headshot has (…/7/people/<hash>.jpg): plain path segments, no query, no
|
||||
# fragment, no dot segment or empty segment (an HTTP client folds `a/../b` and the CDN `a//b` onto one
|
||||
# file). The person-image proxy takes nothing else, so it cannot be made to cache endless spellings of
|
||||
# one file (the image cache has no eviction). Used where a stored headshot is made, served and warmed.
|
||||
_SEGMENT = r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*" # a name, dots only between its parts
|
||||
_PORTRAIT_URL = re.compile(re.escape(PERSON_IMG_HOST) + _SEGMENT + rf"(?:/{_SEGMENT})*")
|
||||
|
||||
|
||||
def is_portrait_url(url: str) -> bool:
|
||||
return _PORTRAIT_URL.fullmatch(url) is not None
|
||||
|
||||
|
||||
def _portrait_keys(url: str) -> tuple[str, str]:
|
||||
"""(the resized portrait's key, the full-size original's key)."""
|
||||
base = "person_" + hashlib.sha1(url.encode()).hexdigest()
|
||||
return f"{base}_{PORTRAIT_BOX}", base
|
||||
|
||||
|
||||
def portrait_cached(url: str) -> tuple[bytes, str] | None:
|
||||
"""The cached portrait: the resized one, else an original kept from a time without Plex."""
|
||||
small, original = _portrait_keys(url)
|
||||
return imgcache.read(small) or imgcache.read(original)
|
||||
|
||||
|
||||
def cache_portrait(plex: PlexClient | None, url: str) -> tuple[bytes, str]:
|
||||
"""Fetch one CDN headshot and cache it. With a Plex client it is resized (the box key; the original
|
||||
is removed); without one the original comes straight from the public CDN and is cached under its own
|
||||
key, so the job still resizes it once Plex is there. Raises PlexError / httpx.HTTPError."""
|
||||
small, original = _portrait_keys(url)
|
||||
if plex is None:
|
||||
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
||||
r = c.get(url)
|
||||
r.raise_for_status()
|
||||
data, ct = r.content, r.headers.get("content-type", "image/jpeg")
|
||||
imgcache.write(original, data, ct)
|
||||
return data, ct
|
||||
data, ct = plex.image_bytes(url, PORTRAIT_BOX, PORTRAIT_BOX)
|
||||
imgcache.write(small, data, ct)
|
||||
imgcache.drop(original)
|
||||
return data, ct
|
||||
|
||||
|
||||
def cdn_url(photo_thumb: str) -> str | None:
|
||||
"""The CDN url inside a stored proxied `/person-image?u=` value (see person_thumb_url)."""
|
||||
_, _, q = photo_thumb.partition("?u=")
|
||||
url = unquote(q)
|
||||
return url if is_portrait_url(url) else None
|
||||
|
||||
|
||||
# Plex failing this many headshots in a row ends a run: it is Plex that is failing, not the files.
|
||||
_PLEX_DOWN_AFTER = 3
|
||||
# The statuses that mean the one file is gone.
|
||||
_DEAD = frozenset({404, 410})
|
||||
|
||||
|
||||
def warm_portraits(db: Session, limit: int) -> dict:
|
||||
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first, so a
|
||||
person's face comes from local disk. Needs Plex (the resize) and the Plex module on; without them
|
||||
nothing is prefetched — full-size originals for every person would cost gigabytes. A headshot Plex
|
||||
answers "not found" for is marked and left for PORTRAIT_RETRY; Plex failing in a row ends the run."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
return {"cached": 0, "skipped": "disabled"}
|
||||
try:
|
||||
plex = PlexClient(db)
|
||||
# silent-ok: no Plex server set up is a setting, not a failure; the run's result says "disabled".
|
||||
except PlexNotConfigured:
|
||||
return {"cached": 0, "skipped": "disabled"}
|
||||
photos: dict[int, str] = {
|
||||
pid: thumb
|
||||
for pid, thumb in db.execute(
|
||||
select(PlexPerson.id, PlexPerson.photo_thumb).where(PlexPerson.photo_thumb.is_not(None))
|
||||
).tuples()
|
||||
if thumb
|
||||
}
|
||||
# `down` counts Plex failing in a row (any answer from Plex breaks it), `plex_failed` in all
|
||||
done = dead = plex_failed = down = 0
|
||||
with plex:
|
||||
for pid in people_by_credits(db, "portrait", None):
|
||||
if done + dead >= limit or down >= _PLEX_DOWN_AFTER:
|
||||
break
|
||||
url = cdn_url(photos.get(pid) or "")
|
||||
if url is None:
|
||||
continue
|
||||
key = _portrait_keys(url)[0]
|
||||
if imgcache.has(key) or imgcache.missed_within(key, PORTRAIT_RETRY):
|
||||
continue
|
||||
try:
|
||||
cache_portrait(plex, url)
|
||||
done, down = done + 1, 0
|
||||
except PlexError as exc:
|
||||
# "Not found" is Plex's answer about this one file (a dead CDN headshot: measured 404,
|
||||
# the CDN itself says 403). Anything else — no answer, a refused token, a 5xx — is
|
||||
# Plex failing, which must neither mark the file nor keep the run going.
|
||||
cause = exc.__cause__
|
||||
if isinstance(cause, httpx.HTTPStatusError) and cause.response.status_code in _DEAD:
|
||||
dead, down = dead + 1, 0 # Plex answered: it is up, the streak is broken
|
||||
imgcache.mark_miss(key)
|
||||
else:
|
||||
plex_failed, down = plex_failed + 1, down + 1
|
||||
log.warning(
|
||||
"a portrait could not be cached",
|
||||
extra=logs.event(
|
||||
"plex.people.portrait_failed",
|
||||
person_id=pid,
|
||||
reason=type(exc.__cause__).__name__,
|
||||
),
|
||||
)
|
||||
failed = dead + plex_failed
|
||||
return {"cached": done, **({"failed": failed} if failed else {})}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Plex text fields as plain text.
|
||||
|
||||
Plex hands a summary over as the agent wrote it, and some agents (a Hungarian TV listing, for one)
|
||||
write HTML: `<p>…</p>` paragraphs and `á`-style entities. The app renders summaries as text,
|
||||
so the tags showed literally on the info page and the hover card, and a card's 300-character cut
|
||||
could land inside one. Every summary and tagline is stored through `plain_text`, and the migration
|
||||
that cleaned the existing rows calls it too, so the two cannot disagree.
|
||||
"""
|
||||
|
||||
import html
|
||||
import re
|
||||
from html.entities import html5
|
||||
|
||||
_BREAK = re.compile(r"<br\s*/?>", re.IGNORECASE)
|
||||
# A paragraph or block boundary: a blank line between the two. Only a tag name ends the match, so a
|
||||
# bare "<" in prose ("a < b", "<3") is left alone.
|
||||
_BLOCK = re.compile(r"</?(?:p|div|li|ul|ol|h[1-6]|blockquote)\b[^>]*>", re.IGNORECASE)
|
||||
_TAG = re.compile(r"</?[a-zA-Z][^>]*>")
|
||||
_BLANKS = re.compile(r"[ \t]*\n[ \t]*\n\s*")
|
||||
|
||||
# A well-formed character reference only. `html.unescape` alone also decodes the legacy forms that
|
||||
# have no semicolon, so prose like "AT¬ice" or "R©" came out as "AT¬ice" and "R©". A named
|
||||
# one counts only when its whole name is an entity: for "¬ice;" `html.unescape` still falls back
|
||||
# to the legacy "¬" inside it.
|
||||
_REF = re.compile(r"&(?:#[0-9]+|#[xX][0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]*);")
|
||||
|
||||
|
||||
def _decode(m: re.Match[str]) -> str:
|
||||
ref = m.group()
|
||||
if ref[1] != "#":
|
||||
return html.unescape(ref) if ref[1:] in html5 else ref
|
||||
if ref[2] in "xX":
|
||||
return html.unescape(ref)
|
||||
# `int()` refuses a decimal run past 4300 digits, and any number past seven significant digits is
|
||||
# beyond U+10FFFF, which the parser turns into U+FFFD anyway.
|
||||
digits = ref[2:-1].lstrip("0") or "0"
|
||||
return "�" if len(digits) > 7 else html.unescape(f"&#{digits};")
|
||||
|
||||
|
||||
def plain_text(value: str | None) -> str | None:
|
||||
"""`value` with its markup gone: line breaks kept as newlines, paragraphs as a blank line,
|
||||
every other tag dropped and entities decoded. None or an all-blank result stays None."""
|
||||
if value is None:
|
||||
return None
|
||||
if "<" in value or "&" in value:
|
||||
value = _BREAK.sub("\n", value)
|
||||
value = _BLOCK.sub("\n\n", value)
|
||||
value = _TAG.sub("", value)
|
||||
value = _REF.sub(_decode, value)
|
||||
value = _BLANKS.sub("\n\n", value)
|
||||
return value.strip() or None
|
||||
+18
-11
@@ -11,6 +11,7 @@ from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import PlexItem, PlexLibrary
|
||||
from app.plex.genres import hu_genre
|
||||
|
||||
TS_CONFIG = "public.unaccent_simple"
|
||||
|
||||
@@ -30,9 +31,9 @@ PLEX_SORT_KEYS: tuple[str, ...] = (
|
||||
"title",
|
||||
)
|
||||
|
||||
# --- Language-aware column pickers (E5 S6) ----------------------------------
|
||||
# --- Language-aware column pickers ----------------------------------
|
||||
#
|
||||
# The HU-overlay twin (E5 S5) stores Hungarian metadata in `*_hu` columns on the SAME EN row. When
|
||||
# The HU-overlay twin stores Hungarian metadata in `*_hu` columns on the SAME EN row. When
|
||||
# the app language is `hu` we serve those, falling back to the EN column where a row has no HU twin.
|
||||
# Two flavours of picker, on purpose:
|
||||
# * DISPLAY / FILTER (`title_col`/`genres_col`): `coalesce(*_hu, *)` — an EN fallback so a row with
|
||||
@@ -40,8 +41,8 @@ PLEX_SORT_KEYS: tuple[str, ...] = (
|
||||
# HU summary directly off the loaded row, so no `summary_col` SQL helper is needed.)
|
||||
# * SEARCH (`vector_col` + `field_recheck`): the raw `*_hu` column coalesced to '' — MIRRORING the
|
||||
# generated `search_vector_hu` definition (which uses `coalesce(title_hu, '')`, NOT an EN
|
||||
# fallback), so the field recheck stays a strict subset of the GIN vector prefilter (the S2
|
||||
# invariant). `people_text` is language-neutral and shared by both vectors.
|
||||
# fallback), so the field recheck stays a strict subset of the GIN vector prefilter (the
|
||||
# field-search invariant). `people_text` is language-neutral and shared by both vectors.
|
||||
|
||||
|
||||
def title_col(model, lang: str):
|
||||
@@ -183,7 +184,13 @@ def meta_filter_conds(model, p: dict, lang: str = "en") -> list:
|
||||
gsel = split_csv(p.get("genres"))
|
||||
if gsel:
|
||||
gcol = genres_col(model, lang)
|
||||
gc = [gcol.contains([g]) for g in gsel]
|
||||
if lang == "hu":
|
||||
# The HU column holds one Hungarian word per genre (plex/genres.py), so a filter saved while
|
||||
# it offered "Action" is read as "Akció" too. The word as picked still matches as well: a row
|
||||
# the sync has not reached since keeps its English (or Plex's mixed) list until it does.
|
||||
gc = [or_(*(gcol.contains([w]) for w in dict.fromkeys((hu_genre(g), g)))) for g in gsel]
|
||||
else:
|
||||
gc = [gcol.contains([g]) for g in gsel]
|
||||
conds.append(and_(*gc) if p.get("genre_mode") == "all" else or_(*gc))
|
||||
crs = split_csv(p.get("content_ratings"))
|
||||
if crs:
|
||||
@@ -209,7 +216,7 @@ def meta_filter_conds(model, p: dict, lang: str = "en") -> list:
|
||||
stds = split_csv(p.get("studios"))
|
||||
if stds: # OR
|
||||
conds.append(model.studio.in_(stds))
|
||||
for kw in split_csv(p.get("keywords")): # AND — TMDB keyword chips from the detail page (E5 S5)
|
||||
for kw in split_csv(p.get("keywords")): # AND — TMDB keyword chips from the detail page
|
||||
conds.append(model.keywords.contains([kw]))
|
||||
csel = split_csv(p.get("collections"))
|
||||
if (
|
||||
@@ -225,7 +232,7 @@ def apply_meta_filters(q, model, p: dict, lang: str = "en"):
|
||||
return q.where(*conds) if conds else q
|
||||
|
||||
|
||||
# --- Playback-class filter (E7) — MOVIE-ONLY (a show is an aggregate with no single class) ---------
|
||||
# --- Playback-class filter — MOVIE-ONLY (a show is an aggregate with no single class) ---------
|
||||
# The 4 classes the feed filter + facet expose, derived from the stored `playable` + `transcode_hw`.
|
||||
# ONE definition (this SQL CASE) so the filter WHERE and the facet counts can't drift: `playable_conds`
|
||||
# filters on it, `facets` groups by it. transcode_hw IS NOT True groups NULL (unprobed) with software.
|
||||
@@ -271,7 +278,7 @@ class LibraryFilters:
|
||||
actors: str | None = None
|
||||
actor_mode: str = "any" # how multiple actors combine: any (OR, default) | all (AND)
|
||||
studios: str | None = None
|
||||
keywords: str | None = None # TMDB keyword chips (E5 S5), CSV; language-neutral
|
||||
keywords: str | None = None # TMDB keyword chips, CSV; language-neutral
|
||||
collections: str | None = None # CSV of collection rating_keys
|
||||
collection_mode: str = "any" # how multiple collections combine: any (OR, default) | all (AND)
|
||||
playable: str | None = None # CSV of playback classes: direct|remux|transcode_hw|transcode_sw
|
||||
@@ -304,7 +311,7 @@ class LibraryFilters:
|
||||
# watch state. The SAME predicate has to be applied in two places — `unified_library` (the grid) and
|
||||
# `facets.wsq` (so the facet counts reflect exactly the visible set) — and they MUST stay identical:
|
||||
# a hidden movie kept out of the grid must not inflate the facets either. These helpers are the one
|
||||
# home for that predicate so the two call sites can't drift (E3 S1; pinned by test_plex_facets_grid_
|
||||
# home for that predicate so the two call sites can't drift (pinned by test_plex_facets_grid_
|
||||
# parity). Callers pass the already-joined state expressions, so this module stays model-agnostic:
|
||||
# * movies — `st` is an aliased PlexState outer-joined on (item_id, user_id).
|
||||
# * shows — `tot`/`wat`/`inp` are the coalesced totals from `rollup.agg_subquery`.
|
||||
@@ -360,9 +367,9 @@ def apply_show_state_filter(query, tot, wat, inp, show: str):
|
||||
def content_libs(db: Session) -> tuple[list[int], list[int]]:
|
||||
"""The BROWSABLE content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
|
||||
source of the browse scope. Excludes HU overlay libraries (`overlay_of` set): those are never
|
||||
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows (E5 S5).
|
||||
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows.
|
||||
|
||||
Lives here, in the query layer, rather than in `routes/plex/_shared.py` where it started: E24's
|
||||
Lives here, in the query layer, rather than in `routes/plex/_shared.py` where it started: the
|
||||
Home surface needs the same scope, and a domain package importing from `routes/` would invert the
|
||||
one-way dependency CLAUDE.md draws. It never touched request state, so nothing was lost by the
|
||||
move."""
|
||||
|
||||
@@ -9,14 +9,15 @@ from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.models import PlexItem, PlexState
|
||||
from app.plex.cards import ShowState
|
||||
|
||||
# Sort sentinel for a rail row whose timestamp is missing, so `key=lambda r: r[0] or NO_TS` reads the
|
||||
# same at every rail that merges two type-homogeneous lists in Python.
|
||||
#
|
||||
# It is NOT a crash fix, whatever the E24 commit that introduced it says. The `(ts is not None, ts)`
|
||||
# It is NOT a crash fix, whatever the commit that introduced it says. The `(ts is not None, ts)`
|
||||
# key it replaced cannot raise: tuple comparison finds the first UNEQUAL element via `==`, so two
|
||||
# `(False, None)` keys compare equal and `None < None` is never reached — verified by execution
|
||||
# during the E24 epic-end review, over every permutation of null/non-null rows. What the sentinel
|
||||
# during a review, over every permutation of null/non-null rows. What the sentinel
|
||||
# buys is one idiom instead of two, and a key that says "missing sorts last" outright rather than
|
||||
# encoding it in a boolean flag. A BARE `r[0]` would genuinely raise, which is the mutation the
|
||||
# rails' tests do guard against.
|
||||
@@ -32,6 +33,11 @@ def show_status(total: int, watched: int, inprog: int) -> str:
|
||||
return "new"
|
||||
|
||||
|
||||
def show_state(total: int, watched: int, inprog: int) -> ShowState:
|
||||
"""`show_status` plus whether an episode holds a resume point (the card's `ShowState`)."""
|
||||
return ShowState(show_status(total, watched, inprog), (inprog or 0) > 0)
|
||||
|
||||
|
||||
def rollup(cards: list[dict]) -> dict:
|
||||
"""From ORDERED episode cards → the aggregate watch status, the "on deck" episode to Resume (the
|
||||
last in-progress one, else the first unwatched), the first episode (for Play from the start), and
|
||||
@@ -92,16 +98,16 @@ def agg_subquery(db: Session, user_id: int, include_resumed: bool = False):
|
||||
)
|
||||
|
||||
|
||||
def state_map(db: Session, user_id: int, show_ids: list[int]) -> dict[int, str]:
|
||||
def state_map(db: Session, user_id: int, show_ids: list[int]) -> dict[int, ShowState]:
|
||||
if not show_ids:
|
||||
return {}
|
||||
rows = db.execute(agg_subquery(db, user_id).where(PlexItem.show_id.in_(show_ids))).all()
|
||||
return {r.show_id: show_status(r.total, r.watched, r.inprog) for r in rows}
|
||||
return {r.show_id: show_state(r.total, r.watched, r.inprog) for r in rows}
|
||||
|
||||
|
||||
def item_state_map(db: Session, user_id: int, item_ids: list[int]) -> dict[int, PlexState]:
|
||||
"""A user's PlexState rows for a set of leaf items (movies or episodes), keyed by item_id — the
|
||||
leaf analogue of `state_map` (which rolls a show's episodes up to a status string). One batched
|
||||
leaf analogue of `state_map` (which rolls a show's episodes up to a `ShowState`). One batched
|
||||
fetch, so the caller serializes cards without a per-card query. Empty in → empty dict (no `IN ()`)."""
|
||||
if not item_ids:
|
||||
return {}
|
||||
|
||||
+54
-25
@@ -15,7 +15,7 @@ HLS muxer does the segmentation (the only reliable way to cut a stream-copy at k
|
||||
|
||||
Sessions live in this (single) API process; a periodic reaper kills idle ones and frees the temp
|
||||
segments. The remux is CPU-light (video copy + a tiny audio transcode), so it runs fine on the
|
||||
CPU-only prod host; only P3's full transcode is CPU-heavy.
|
||||
CPU-only prod host; only the full transcode is CPU-heavy.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -35,13 +35,13 @@ from app.config import settings
|
||||
from app.downloads.edit import encode_thread_cap
|
||||
from app.models import PlexItem
|
||||
from app.plex import paths
|
||||
from app.plex.sync import HW_DECODABLE_CODECS
|
||||
from app.plex.sync import HDR_TRANSFERS, HW_DECODABLE_CODECS
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_HLS_ROOT = (
|
||||
Path(settings.plex_hls_dir)
|
||||
if settings.plex_hls_dir
|
||||
else Path(settings.download_root) / ".plex-hls"
|
||||
else Path(settings.files_root) / ".plex-hls"
|
||||
)
|
||||
_SEG_SECONDS = 6
|
||||
# Cold-start warm-up. A freshly-spawned ffmpeg has produced only the first segment (or two) by the
|
||||
@@ -120,8 +120,8 @@ class HlsSession:
|
||||
# Unguessable per-session id, carried in the URLs the client is handed. A seek-restart
|
||||
# replaces the viewer's registered session for the item — and the old player's next request
|
||||
# for `seg_12.ts`, resolved by (viewer, rating_key) alone, was answered 200 with the NEW session's twelfth segment, produced from a different offset
|
||||
# under `-copyts`. Wrong bytes, not an error, so the client's reclaim never fired on it
|
||||
# (deep-review round 5). A request naming a session that is no longer registered is a 404,
|
||||
# under `-copyts`. Wrong bytes, not an error, so the client's reclaim never fired on it.
|
||||
# A request naming a session that is no longer registered is a 404,
|
||||
# which the player recovers from by restarting at the playhead.
|
||||
self.id = secrets.token_urlsafe(9)
|
||||
self.key = key # the item's rating_key (what the logs name)
|
||||
@@ -203,7 +203,8 @@ _HW_DECODABLE = HW_DECODABLE_CODECS
|
||||
# plain scale_vaapi=format=nv12 path (Gen 9.5 VPP refuses it) — it needs the OpenCL tonemap graph
|
||||
# (_venc_opts tonemap branch) when the OpenCL runtime is present, else software libx264. A 10-bit source
|
||||
# WITHOUT one of these transfers is SDR, not HDR, and must not be tone-mapped. See _probe_video_kind.
|
||||
_HDR_TRANSFERS = {"smpte2084", "arib-std-b67"}
|
||||
# THE single source is plex.sync (a clip's tonemap reads it too).
|
||||
_HDR_TRANSFERS = HDR_TRANSFERS
|
||||
|
||||
|
||||
# The three source classes that drive the transcode-pipeline choice (see _probe_video_kind).
|
||||
@@ -245,6 +246,9 @@ def _probe_video_kind(src: Path) -> str:
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
# A non-zero exit is a failed probe (CalledProcessError, a SubprocessError → the branch
|
||||
# below), not an answer: whatever it printed before refusing is not the file's truth.
|
||||
check=True,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
# The safe kind is a real answer, but it is the answer for an 8-bit SDR source — a blind
|
||||
@@ -410,7 +414,7 @@ def _venc_opts(
|
||||
def _render_node_present() -> bool:
|
||||
"""Whether the VAAPI render node is available — i.e. `/dev/dri` was passed into the container.
|
||||
Present on prod after the iGPU enablement; absent on the CPU-only dev box and in the localdev
|
||||
Docker. Its own function so the selection is monkeypatchable in tests and one place for S1c's knob."""
|
||||
Docker. Its own function so the selection is monkeypatchable in tests and the one place to change it."""
|
||||
return Path(_VAAPI_DEVICE).exists()
|
||||
|
||||
|
||||
@@ -502,6 +506,9 @@ def _probe_audio_langs(src: Path) -> list[str]:
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
# A non-zero exit is a failed probe (CalledProcessError, a SubprocessError → the branch
|
||||
# below), not an answer: whatever it printed before refusing is not the file's truth.
|
||||
check=True,
|
||||
)
|
||||
return [ln.strip() for ln in out.stdout.splitlines()]
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
@@ -570,7 +577,7 @@ def _ffmpeg_cmd(
|
||||
entry += ",default:yes"
|
||||
var.append(entry)
|
||||
# The name patterns come from the vocabulary functions, with ffmpeg's own placeholders passed
|
||||
# through them — so the producer and the parser cannot spell the layout differently (round 7).
|
||||
# through them — so the producer and the parser cannot spell the layout differently.
|
||||
args += _HLS_TAIL + [
|
||||
"-master_pl_name",
|
||||
_MASTER_PL,
|
||||
@@ -615,6 +622,9 @@ def _probe_first_pts(seg: Path) -> float | None:
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
# A non-zero exit is a failed probe (CalledProcessError, a SubprocessError → the branch
|
||||
# below), not an answer: whatever it printed before refusing is not the file's truth.
|
||||
check=True,
|
||||
)
|
||||
for line in out.stdout.splitlines():
|
||||
tok = line.strip().strip(",")
|
||||
@@ -735,9 +745,9 @@ def start_session(
|
||||
# `replace_only` makes the supersede identity-aware for the runtime fallback: the whole
|
||||
# check-and-replace happens under ONE _lock, so a newer same-key session (a concurrent
|
||||
# seek-restart) installed between the fallback's decision and here is never clobbered. When the
|
||||
# registered session is no longer `replace_only`, we abort and return None (the caller yields to
|
||||
# whoever is registered now). The INITIAL spawn passes replace_only=None and always supersedes —
|
||||
# that is the seek-restart's intended last-writer-wins.
|
||||
# registered session is no longer `replace_only`, we abort and return None, and the caller
|
||||
# answers with its own dead session — never the newer one, whose id a stale requester would
|
||||
# stop. The INITIAL spawn never gets None — see its call site.
|
||||
# Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and
|
||||
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
||||
preset, crf, max_height = "veryfast", 20, 0
|
||||
@@ -814,6 +824,7 @@ def start_session(
|
||||
return s
|
||||
|
||||
s = _spawn(*_select_encode(db, item, transcode, src))
|
||||
assert s is not None # replace_only=None always supersedes: a seek-restart's last-writer-wins
|
||||
# Wait for the first video segment outside the lock (this blocks on ffmpeg, and we must not hold up
|
||||
# other sessions). Pass the proc so a HW session that dies at VAAPI init fails fast (not after 25s).
|
||||
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
||||
@@ -831,12 +842,13 @@ def start_session(
|
||||
)
|
||||
respawned = _spawn(False, False, replace_only=s)
|
||||
if respawned is None:
|
||||
# Superseded/culled mid-fallback — yield whoever is registered now (a newer seek-restart), or
|
||||
# our own now-dead session if the key was cap-culled. Never None here: the POST /session route
|
||||
# turns a None return into a "Media file not found on disk" 404 — dishonest for a file that
|
||||
# exists. A (dead) session instead degrades to a retryable 404 ("No active playback session" /
|
||||
# "Not available yet") the client re-requests.
|
||||
return current_session(user_id, key) or s
|
||||
# Superseded/culled mid-fallback — answer with our own now-dead session. Never the newer one
|
||||
# registered meanwhile: this answer goes to the request that LOST, whose player drops it as
|
||||
# stale and stops the session it names (`stop_session`), which stopped the playback that won.
|
||||
# Never None either: the POST /session route turns a None return into a "Media file not found
|
||||
# on disk" 404 — dishonest for a file that exists. A dead session instead degrades to a
|
||||
# retryable 404 ("No active playback session" / "Not available yet") the client re-requests.
|
||||
return s
|
||||
s = respawned
|
||||
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
||||
if ready:
|
||||
@@ -878,7 +890,7 @@ def current_session(user_id: int, key: str) -> HlsSession | None:
|
||||
# The names ffmpeg writes into a session directory, in ONE place — and "one place" means the argv
|
||||
# BUILDS its patterns from these same functions (`segment_name("%d", "%v")`), not that a comment
|
||||
# says so. Getting this wrong is invisible in both directions: a name we failed to anticipate used
|
||||
# to fall back to "wait and see" (round 6's HIGH), and now that the rule is positive it falls back
|
||||
# to fall back to "wait and see", and now that the rule is positive it falls back
|
||||
# to a 404 instead — so the producer and the parser have to be literally the same spelling.
|
||||
#
|
||||
# MEASURED (ffmpeg 7.1.5 in the API container — the binary that actually writes these files — and
|
||||
@@ -892,7 +904,7 @@ def current_session(user_id: int, key: str) -> HlsSession | None:
|
||||
# SESSION and not to HLS in general: a single-audio session can never write `master.m3u8`,
|
||||
# `stream_*.m3u8` or any `seg_<variant>_<n>.ts`, and a multi-audio one can never write `index.m3u8`
|
||||
# or `seg_<n>.ts`. Accepting the union meant a hopeless name of the other shape was waited out for
|
||||
# the full playlist/segment timeout — the threadpool parking this guard exists to end (round 7).
|
||||
# the full playlist/segment timeout — the threadpool parking this guard exists to end.
|
||||
_MASTER_PL = "master.m3u8"
|
||||
_INDEX_PL = "index.m3u8"
|
||||
_SEG_NAME_RE = re.compile(r"^seg_(?:([0-9a-z]{1,4})_)?(\d{1,9})\.ts$")
|
||||
@@ -944,12 +956,12 @@ def artifact_may_arrive(s: "HlsSession", filename: str) -> bool:
|
||||
name belonging to the other shape is as hopeless here as a random string. Reading those as "an
|
||||
HLS name, so wait and see" left the whole class reachable — measured through the real route
|
||||
against a live single-audio session, `master.m3u8` and `stream_q7z.m3u8` were 404s after 30.0 s
|
||||
and `seg_a0_0.ts` / `seg_zzzz_0.ts` after 20.1 s each (deep-review round 7).
|
||||
and `seg_a0_0.ts` / `seg_zzzz_0.ts` after 20.1 s each.
|
||||
|
||||
Saying no at once matters because the route that waits is a SYNC one: it holds a thread off the
|
||||
~40-slot pool, and a pooled DB connection with it, for the whole timeout. An ordinary seek past
|
||||
the produced region was enough to park one for 20 s, and forty concurrent requests for a name
|
||||
that will never exist parked the whole API (deep-review round 5). Round 5's guard read only
|
||||
that will never exist parked the whole API. An earlier guard read only
|
||||
single-audio segment names, so both halves of that were still fully reachable: every multi-audio
|
||||
name fell through, and so did any `.m3u8`/`.vtt` name at all."""
|
||||
if filename in playlist_names(s.variants):
|
||||
@@ -1084,15 +1096,15 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int:
|
||||
A finished session is also what a PAUSED viewer has: a remux runs far faster than realtime, so
|
||||
ffmpeg is done a minute or two in and writes ENDLIST; hls.js then stops asking for the playlist,
|
||||
and with its forward buffer full a paused player fetches nothing at all. With the 30 s rule that
|
||||
killed a one-minute pause, and playback died ~30 s after resuming (deep-review round 4).
|
||||
killed a one-minute pause, and playback died ~30 s after resuming.
|
||||
|
||||
What makes the short rule safe again is that fetching is no longer the only signal: a player
|
||||
parked MID-ITEM checkpoints its position every 10 s, and that route calls `touch_session`, so a
|
||||
viewer who is merely paused says so far more often than this rule asks (round 5). The heartbeat
|
||||
viewer who is merely paused says so far more often than this rule asks. The heartbeat
|
||||
stops once the item is finished or before a position is known (`usePlexProgress.flush`) — both
|
||||
states where reaping is correct, and where the player's 404 → reclaim recovers a viewer who
|
||||
scrubs back in anyway. Do not restate this as "every 10 s whether or not it is playing": round 6
|
||||
found that claim in two comments and not in the code. Round 4's blanket
|
||||
scrubs back in anyway. Do not restate this as "every 10 s whether or not it is playing":
|
||||
a review found that claim in two comments and not in the code. An earlier blanket
|
||||
`reap_abandoned` bought the same safety by keeping every finished session for ten minutes — each
|
||||
one a whole segmented copy of the film on a volume shared with the download center."""
|
||||
now = time.time()
|
||||
@@ -1108,6 +1120,23 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int:
|
||||
return dropped
|
||||
|
||||
|
||||
def stop_session(user_id: int, key: str, session_id: str) -> bool:
|
||||
"""Stop this viewer's session for `key` now, because the player that asked for it is gone (left
|
||||
the item, closed, unloaded the page). Left to `reap_idle`, a transcode ran on to the end of the
|
||||
film first — measured: ~4 cores for a minute after leaving a resumed 480p film.
|
||||
|
||||
Only while `session_id` is still the registered one: the player names the session it played, and
|
||||
a seek-restart or a second tab may have replaced it meanwhile — that newer session is somebody's
|
||||
current playback, not this request's to end."""
|
||||
with _lock:
|
||||
s = _sessions.get((user_id, key))
|
||||
if s is None or s.id != session_id:
|
||||
return False
|
||||
_kill(s)
|
||||
_sessions.pop((user_id, key), None)
|
||||
return True
|
||||
|
||||
|
||||
def touch_session(user_id: int, key: str) -> None:
|
||||
"""Say this viewer is still there for `key`, without fetching anything. `last_access` used to move
|
||||
only on an HLS file fetch, which a paused player does not do — so the only way to keep a paused
|
||||
|
||||
@@ -32,6 +32,7 @@ window still took the full 145 s and emitted the whole track), so windows are ex
|
||||
zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
||||
"""
|
||||
|
||||
import codecs
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
@@ -161,7 +162,8 @@ def ffmpeg_to_vtt(src: str, sub_ord: int | None) -> str:
|
||||
"webvtt",
|
||||
"pipe:1",
|
||||
]
|
||||
r = subprocess.run(cmd, capture_output=True, timeout=EXTRACT_TIMEOUT_S)
|
||||
# check=False: the exit status is read on the next line, with ffmpeg's own stderr as the reason.
|
||||
r = subprocess.run(cmd, capture_output=True, timeout=EXTRACT_TIMEOUT_S, check=False)
|
||||
if r.returncode != 0 or not r.stdout:
|
||||
raise PlexError(
|
||||
(r.stderr or b"").decode("utf-8", "replace")[:200] or "ffmpeg subtitle convert failed"
|
||||
@@ -217,7 +219,11 @@ _FULL_END_S = 1_000_000_000.0 # stand-in for "to the end of the file" (JSON can
|
||||
|
||||
|
||||
def cache_key(rating_key: str, sub_ord: int) -> str:
|
||||
return f"sub_{rating_key}_{sub_ord}"
|
||||
# Versioned whenever `decode_subtitle` changes what a track decodes to — `sub2_` when legacy code
|
||||
# pages were cached as U+FFFD, `sub3_` when a stray byte, an untagged file or Serbian still went
|
||||
# to the wrong code page. The cache has no eviction, so every track is read again once rather
|
||||
# than a broken one being served for good; the older files are just left behind.
|
||||
return f"sub3_{rating_key}_{sub_ord}"
|
||||
|
||||
|
||||
def _vtt_path(key: str) -> Path:
|
||||
@@ -904,15 +910,90 @@ def _full_pass_start(rating_key: str, targets: dict[int, int]) -> float:
|
||||
return max(0.0, float(f"{min(ends) - WINDOW_PRE_S:.3f}"))
|
||||
|
||||
|
||||
# The legacy code page a subtitle in this language (Plex's ISO 639-2 `languageCode`) was most likely
|
||||
# saved in. Serbian sits here, not with Cyrillic: its releases are overwhelmingly Latin script in
|
||||
# Windows-1250, like Croatian and Bosnian.
|
||||
_CP1250 = {
|
||||
"hun",
|
||||
"ces",
|
||||
"cze",
|
||||
"pol",
|
||||
"slk",
|
||||
"slo",
|
||||
"slv",
|
||||
"hrv",
|
||||
"bos",
|
||||
"srp",
|
||||
"ron",
|
||||
"rum",
|
||||
"sqi",
|
||||
"alb",
|
||||
}
|
||||
_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac"}
|
||||
# Western European languages keep Windows-1252 (cp1250 has no à/è/ñ). An UNTAGGED file falls to
|
||||
# Windows-1250: its ASCII and á/é/í/ó/ö/ú/ü match Windows-1252 byte for byte, and it also gets ő/ű
|
||||
# right, which Windows-1252 turns into õ/û — the library this reads is mostly Hungarian.
|
||||
_CP1252 = {
|
||||
"eng",
|
||||
"fre",
|
||||
"fra",
|
||||
"ger",
|
||||
"deu",
|
||||
"spa",
|
||||
"ita",
|
||||
"por",
|
||||
"dut",
|
||||
"nld",
|
||||
"swe",
|
||||
"dan",
|
||||
"nor",
|
||||
"fin",
|
||||
"ice",
|
||||
"isl",
|
||||
"cat",
|
||||
}
|
||||
# A file that is almost all valid UTF-8 is UTF-8 with a few stray bytes (a pasted Windows-1252 quote),
|
||||
# not a legacy file: accents in a legacy code page are nearly never valid UTF-8 sequences, so a real
|
||||
# legacy file has hardly any. At least this many valid multi-byte characters per invalid byte.
|
||||
_UTF8_VALID_PER_STRAY = 10
|
||||
|
||||
|
||||
def decode_subtitle(raw: bytes, language_code: str | None) -> str:
|
||||
"""A sidecar subtitle file as text. SubRip carries no charset, and a Hungarian release is
|
||||
typically Windows-1250: decoded as UTF-8 with `replace`, every ő/ű/é came out as U+FFFD (found
|
||||
live, 1391 of them in one episode). So: a BOM decides; else valid UTF-8 stays UTF-8 (a legacy
|
||||
code page with accents is almost never valid UTF-8), and so does NEARLY valid UTF-8 — one stray
|
||||
byte must not send a whole UTF-8 file to a legacy code page; else the language's legacy code page."""
|
||||
if raw.startswith(codecs.BOM_UTF8):
|
||||
return raw[len(codecs.BOM_UTF8) :].decode("utf-8", "replace")
|
||||
if raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
|
||||
return raw.decode("utf-16", "replace")
|
||||
try:
|
||||
return raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
# silent-ok: not strictly UTF-8 — nearly-UTF-8 or a legacy code page, told apart below.
|
||||
pass
|
||||
lenient = raw.decode("utf-8", "replace")
|
||||
stray = lenient.count("�")
|
||||
valid = sum(1 for ch in lenient if ch > "" and ch != "�")
|
||||
if valid >= stray * _UTF8_VALID_PER_STRAY:
|
||||
return lenient
|
||||
lang = (language_code or "").lower()
|
||||
legacy = "cp1251" if lang in _CP1251 else "cp1252" if lang in _CP1252 else "cp1250"
|
||||
return raw.decode(legacy, "replace")
|
||||
|
||||
|
||||
def _sidecar_vtt(db: Session, stream: dict) -> str:
|
||||
"""An external subtitle file, fetched through Plex and converted to WebVTT."""
|
||||
"""An external subtitle file, fetched through Plex and converted to WebVTT. Its bytes are made
|
||||
UTF-8 first (`decode_subtitle`), on the ffmpeg path too: ffmpeg reads a text subtitle as UTF-8."""
|
||||
with PlexClient(db) as plex:
|
||||
raw = plex.raw_get(stream["key"])
|
||||
text = decode_subtitle(raw, stream.get("languageCode"))
|
||||
codec = (stream.get("codec") or stream.get("format") or "").lower()
|
||||
if codec in ("srt", "subrip", ""):
|
||||
return srt_to_vtt(raw.decode("utf-8", "replace"))
|
||||
return srt_to_vtt(text)
|
||||
with tempfile.NamedTemporaryFile(suffix="." + (codec or "sub"), delete=False) as fh:
|
||||
fh.write(raw)
|
||||
fh.write(text.encode("utf-8"))
|
||||
tmp = fh.name
|
||||
try:
|
||||
return ffmpeg_to_vtt(tmp, None)
|
||||
|
||||
+76
-93
@@ -12,7 +12,7 @@ on a NON-empty listing — a section that returns zero items is left untouched,
|
||||
glitch that yields an empty page can't wipe a library (the metadata is cheap to re-sync; an accidental
|
||||
mass-delete is the only real risk). Library-level reconcile (dropping a whole removed section, the
|
||||
dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are fetched lazily on the
|
||||
item-detail/player path (P2), not here.
|
||||
item-detail/player path, not here.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
@@ -22,9 +22,11 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||
from app.plex import detail, people
|
||||
from app.plex import detail, overlay, people
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.genres import hu_genres
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.plex.plaintext import plain_text
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
||||
@@ -62,9 +64,11 @@ def classify_playable(vcodec: str | None, acodec: str | None, container: str | N
|
||||
# outside it (AV1, MPEG-4 ASP/XviD…) is software-only even on the iGPU (it can't decode it to feed its
|
||||
# own encoder).
|
||||
HW_DECODABLE_CODECS = frozenset({"h264", "hevc", "h265", "vp9", "vc1", "mpeg2video"})
|
||||
# Color-transfer values marking an HDR source (PQ / HLG). A 10-bit/HDR source breaks the VAAPI
|
||||
# scale=nv12 path on Gen 9.5 (see plex.stream) → software transcode, so it counts as NOT HW-capable.
|
||||
_HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"})
|
||||
# Color-transfer values marking an HDR source (PQ / HLG). THE single source of this set: plex.stream
|
||||
# (the tonemap choice) and downloads.clip (a clip's tonemap) import it here, so a transfer added for
|
||||
# one is tonemapped by all. A 10-bit/HDR source breaks the VAAPI scale=nv12 path on Gen 9.5 (see
|
||||
# plex.stream) → software transcode, so it counts as NOT HW-capable.
|
||||
HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"})
|
||||
|
||||
|
||||
def transcode_hw_capable(
|
||||
@@ -105,7 +109,7 @@ def _video_stream_props(meta: dict) -> tuple[int | None, bool]:
|
||||
# whole library would drown the sync's own lines.
|
||||
except (TypeError, ValueError):
|
||||
depth = None
|
||||
hdr = str(s.get("colorTrc") or "").lower() in _HDR_TRANSFERS
|
||||
hdr = str(s.get("colorTrc") or "").lower() in HDR_TRANSFERS
|
||||
return depth, hdr
|
||||
return None, False
|
||||
|
||||
@@ -149,7 +153,7 @@ _GUID_PREFERENCE = ("tmdb", "imdb", "tvdb")
|
||||
|
||||
|
||||
def _canonical_guid(meta: dict) -> str | None:
|
||||
"""The canonical external id used to pair an EN title with its HU-overlay twin (E5 S5). Reads the
|
||||
"""The canonical external id used to pair an EN title with its HU-overlay twin. Reads the
|
||||
`Guid` array (present only with includeGuids=1 in the listing), preferring tmdb > imdb > tvdb,
|
||||
normalised to '<provider>:<id>' (e.g. 'tmdb:12345'). None when no known provider id is present."""
|
||||
found: dict[str, str] = {}
|
||||
@@ -192,20 +196,6 @@ def _enabled_section_keys(db: Session) -> set[str] | None:
|
||||
return keys or None
|
||||
|
||||
|
||||
def _overlay_map(db: Session) -> dict[str, str]:
|
||||
"""The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
|
||||
('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
|
||||
Malformed pairs are skipped."""
|
||||
raw = sysconfig.get_str(db, "plex_overlay_map")
|
||||
out: dict[str, str] = {}
|
||||
for pair in raw.split(","):
|
||||
hu, sep, en = pair.partition(":")
|
||||
hu, en = hu.strip(), en.strip()
|
||||
if sep and hu and en:
|
||||
out[hu] = en
|
||||
return out
|
||||
|
||||
|
||||
def sync(db: Session) -> dict:
|
||||
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
@@ -221,7 +211,7 @@ def sync(db: Session) -> dict:
|
||||
"pruned": 0,
|
||||
}
|
||||
wanted = _enabled_section_keys(db)
|
||||
overlay_map = _overlay_map(db) # {huKey: enKey}
|
||||
overlay_map = overlay.overlay_map(db) # {huKey: enKey}
|
||||
overlay_keys = set(overlay_map)
|
||||
try:
|
||||
with PlexClient(db) as plex:
|
||||
@@ -265,13 +255,13 @@ def sync(db: Session) -> dict:
|
||||
lib.metadata_lang = "hu"
|
||||
lib.overlay_of = en_lib.id
|
||||
db.flush()
|
||||
_sync_overlay(db, plex, lib, en_lib.id, stats)
|
||||
overlay.sync_overlay(db, plex, lib, en_lib.id, stats)
|
||||
db.commit()
|
||||
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
|
||||
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
||||
try:
|
||||
stats["people_enriched"] = _enrich_people_batch(db, plex)
|
||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
||||
except Exception: # best-effort; never fail the sync over a backfill
|
||||
# With the traceback: a blind catch over a batch that fetches AND writes can be
|
||||
# a Plex shape change, a DB error or our own bug, and `%s` told them apart badly.
|
||||
log.exception("the Plex people-enrich batch failed")
|
||||
@@ -280,7 +270,7 @@ def sync(db: Session) -> dict:
|
||||
# successive runs. Only HW-decodable-codec transcodes are probed — the rest are SW by codec.
|
||||
try:
|
||||
stats["media_probed"] = _probe_media_batch(db, plex)
|
||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
||||
except Exception: # best-effort; never fail the sync over a backfill
|
||||
log.exception("the Plex media-probe batch failed")
|
||||
# silent-ok: the return value IS the report. `{"skipped": …}` goes to the scheduler, which
|
||||
# logs the job result, and to the admin route, which answers it to the caller.
|
||||
@@ -294,7 +284,7 @@ def sync(db: Session) -> dict:
|
||||
# appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata.
|
||||
try:
|
||||
stats["people_pruned"] = people.prune_orphan_people(db)
|
||||
except Exception: # noqa: BLE001 — best-effort housekeeping; never fail the sync over it
|
||||
except Exception: # best-effort housekeeping; never fail the sync over it
|
||||
log.exception("the Plex people prune failed")
|
||||
log.info("Plex sync done: %s", stats)
|
||||
return stats
|
||||
@@ -431,7 +421,10 @@ def _paginate(plex: PlexClient, key: str, item_type: int, progress: dict | None
|
||||
if progress is not None:
|
||||
# A truthy `total` we reached = a trustworthy complete listing. `total` unknown/zero-with-rows
|
||||
# (a Plex that doesn't report totalSize) is treated as NOT complete, so prune stays conservative.
|
||||
# `total` itself is kept for a caller to whom a server that SAID 0 is news it can trust (the
|
||||
# watch reconcile); an empty mirror or overlay must not read it so.
|
||||
progress["complete"] = bool(total) and start >= total
|
||||
progress["total"] = total
|
||||
|
||||
|
||||
# The section listing exposes at most the top ~20 cast (Plex's own cap); a stored cast longer than this
|
||||
@@ -448,12 +441,16 @@ def _apply_facet_fields(row, meta: dict) -> None:
|
||||
row.studio = (meta.get("studio") or None) and str(meta.get("studio"))[:255]
|
||||
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
|
||||
row.genres = _tags(meta, "Genre")
|
||||
# Every row has Hungarian genres, not only one with an HU overlay twin (the overlay pass, which runs
|
||||
# after this, puts the twin's own list over it): the HU genre facets and filter read this column,
|
||||
# and an English word there could never match a filter read in Hungarian.
|
||||
row.genres_hu = hu_genres(row.genres)
|
||||
row.directors = _tags(meta, "Director")
|
||||
# The listing carries only the top ~20 cast, so re-applying it would DOWNGRADE an already-enriched
|
||||
# full cast (up to 24, grown by the rich backfill/info-page enrich), silently breaking the
|
||||
# person-page cast filter for names past the top ~20. The watermark gate below (`people_enriched_at
|
||||
# is None`) normally protects this, but it is NOT enough on its own: a migration that resets the
|
||||
# watermark re-opens the regression (0065 nulled it, which is exactly how the S5b fix got undone in
|
||||
# watermark re-opens the regression (0065 nulled it, which is exactly how a fix got undone in
|
||||
# prod). So guard STRUCTURALLY too, by LENGTH against the listing cap: a stored cast LONGER than the
|
||||
# cap can only be an enriched fuller list, so keep it (robust to any watermark reset). A stored cast
|
||||
# at or below the cap is itself listing-sourced, so take the fresh listing — which tracks Plex cast
|
||||
@@ -474,7 +471,7 @@ def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
|
||||
# carries none (a server ignoring includeGuids mustn't wipe the overlay's only match key).
|
||||
row.guid = _canonical_guid(meta) or row.guid
|
||||
row.title = (meta.get("title") or "").strip() or "Untitled"
|
||||
row.summary = meta.get("summary")
|
||||
row.summary = plain_text(meta.get("summary"))
|
||||
row.year = meta.get("year")
|
||||
row.duration_s = _dur_s(meta)
|
||||
row.thumb_key = meta.get("thumb")
|
||||
@@ -582,7 +579,7 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
||||
_canonical_guid(meta) or sh.guid
|
||||
) # EN<->HU pairing key; keep prior if listing has none
|
||||
sh.title = (meta.get("title") or "").strip() or "Untitled"
|
||||
sh.summary = meta.get("summary")
|
||||
sh.summary = plain_text(meta.get("summary"))
|
||||
sh.year = meta.get("year")
|
||||
sh.thumb_key = meta.get("thumb")
|
||||
sh.art_key = meta.get("art")
|
||||
@@ -655,59 +652,6 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
||||
_prune_stale(db, shows, shows_seen, stats, complete=shows_progress.get("complete", False))
|
||||
|
||||
|
||||
def _sync_overlay(
|
||||
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
||||
) -> None:
|
||||
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/genres differ
|
||||
by language; the language-neutral people_text feeds both search vectors.
|
||||
|
||||
This is a full RECONCILE: every guid-bearing EN row's HU fields are cleared first, then the HU
|
||||
section repopulates the matches — so an EN row whose HU twin vanished (or was re-guid'd) falls back
|
||||
to EN instead of keeping stale Hungarian metadata. HU items with no EN match are skipped."""
|
||||
if lib.kind == "movie":
|
||||
rows = db.execute(
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.guid.is_not(None),
|
||||
)
|
||||
).scalars()
|
||||
item_type = _TYPE_MOVIE
|
||||
else:
|
||||
rows = db.execute(
|
||||
select(PlexShow).where(PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None))
|
||||
).scalars()
|
||||
item_type = _TYPE_SHOW
|
||||
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
||||
by_guid: dict[str, list] = {}
|
||||
for r in rows:
|
||||
by_guid.setdefault(r.guid, []).append(r)
|
||||
# Reconcile step: clear HU fields on all guid-bearing rows. Only rows that actually held HU data
|
||||
# become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is cheap; matches are re-set below.
|
||||
for group in by_guid.values():
|
||||
for row in group:
|
||||
row.title_hu = row.summary_hu = row.genres_hu = None
|
||||
seen: set[str] = (
|
||||
set()
|
||||
) # first HU entry for a guid wins; a dup HU item can't overwrite or double-count
|
||||
for meta in _paginate(plex, lib.plex_key, item_type):
|
||||
g = _canonical_guid(meta)
|
||||
targets = by_guid.get(g) if g else None
|
||||
if not targets or g in seen:
|
||||
continue
|
||||
seen.add(g)
|
||||
title_hu = (meta.get("title") or "").strip() or None
|
||||
summary_hu = meta.get("summary")
|
||||
genres_hu = _tags(meta, "Genre")
|
||||
for row in targets:
|
||||
row.title_hu = title_hu
|
||||
row.summary_hu = summary_hu
|
||||
row.genres_hu = genres_hu
|
||||
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
||||
db.flush()
|
||||
|
||||
|
||||
def _upsert_collection(
|
||||
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
|
||||
) -> PlexCollection:
|
||||
@@ -718,7 +662,7 @@ def _upsert_collection(
|
||||
db.add(col)
|
||||
col.library_id = lib_id
|
||||
col.title = (meta.get("title") or "").strip() or "Untitled"
|
||||
col.summary = meta.get("summary")
|
||||
col.summary = plain_text(meta.get("summary"))
|
||||
col.thumb_key = meta.get("thumb")
|
||||
col.child_count = meta.get("childCount")
|
||||
col.smart = str(meta.get("smart")) == "1"
|
||||
@@ -733,9 +677,11 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
||||
meta = plex.metadata(rk) or {}
|
||||
col = _upsert_collection(db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta)
|
||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||
new_members = {
|
||||
str(c.get("ratingKey")) for c in plex.collection_children(rk) if c.get("ratingKey")
|
||||
}
|
||||
children = _believed_children(plex, rk, meta)
|
||||
if children is None:
|
||||
db.flush()
|
||||
return col
|
||||
new_members = {str(c.get("ratingKey")) for c in children if c.get("ratingKey")}
|
||||
current = db.execute(select(Model).where(Model.collection_keys.contains([rk]))).scalars().all()
|
||||
for row in current: # drop rk from rows that are no longer members
|
||||
if row.rating_key not in new_members:
|
||||
@@ -748,6 +694,22 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
||||
return col
|
||||
|
||||
|
||||
def _believed_children(plex: PlexClient, rk: str, meta: dict) -> list[dict] | None:
|
||||
"""A collection's members, or None when the answer is not believed: no members from a collection
|
||||
whose own listing counts some (`childCount`) is a glitch, not a collection emptied — the same
|
||||
empty-answer rule as the collection listing's. Rebuilding on it showed the
|
||||
collection empty until the next sync."""
|
||||
children = plex.collection_children(rk)
|
||||
count = str(meta.get("childCount") or "0")
|
||||
if children or not count.isdigit() or int(count) == 0:
|
||||
return children
|
||||
log.warning(
|
||||
"Plex listed no members for a non-empty collection; keeping its membership",
|
||||
extra=logs.event("plex.sync.collection_children_empty", collection=rk),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
||||
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
|
||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||
@@ -759,6 +721,19 @@ def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
||||
db.flush()
|
||||
|
||||
|
||||
def _rebuild_membership(
|
||||
db: Session, lib: PlexLibrary, membership: dict[str, list[str]], kept: set[str]
|
||||
) -> None:
|
||||
"""Rebuild collection_keys on every member row of this library (clear + set), carrying over the
|
||||
keys of a collection whose empty answer was not believed (`_believed_children`)."""
|
||||
for Model in (PlexItem, PlexShow):
|
||||
for row in db.execute(select(Model).where(Model.library_id == lib.id)).scalars():
|
||||
held = [k for k in (row.collection_keys or []) if k in kept]
|
||||
keys = [*membership.get(row.rating_key, []), *held]
|
||||
row.collection_keys = list(dict.fromkeys(keys)) or None
|
||||
db.flush()
|
||||
|
||||
|
||||
def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
||||
"""Mirror a library's collections + membership. Reads never touch Plex afterwards. Membership is
|
||||
written as `collection_keys` on member movies (plex_items) / shows (plex_shows). A full reconcile:
|
||||
@@ -770,7 +745,17 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
||||
select(PlexCollection).where(PlexCollection.library_id == lib.id)
|
||||
).scalars()
|
||||
}
|
||||
if not cols and existing:
|
||||
# The catalog's own rule (see the module doc): an empty listing while we hold rows is a Plex
|
||||
# glitch, not a library emptied of collections. Reconciling on it dropped every collection
|
||||
# (its user-set `editable` flag with it) and every member's keys.
|
||||
log.warning(
|
||||
"Plex listed no collections for a library that has some; keeping them this run",
|
||||
extra=logs.event("plex.sync.collections_empty", library=lib.plex_key),
|
||||
)
|
||||
return
|
||||
seen: set[str] = set()
|
||||
kept: set[str] = set() # collections whose empty children answer is not believed
|
||||
membership: dict[str, list[str]] = {} # member rating_key -> [collection rating_key, ...]
|
||||
for meta in cols:
|
||||
rk = str(meta.get("ratingKey") or "")
|
||||
@@ -778,7 +763,10 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
||||
continue
|
||||
_upsert_collection(db, existing.get(rk), rk, lib.id, meta)
|
||||
seen.add(rk)
|
||||
for child in plex.collection_children(rk):
|
||||
children = _believed_children(plex, rk, meta)
|
||||
if children is None:
|
||||
kept.add(rk)
|
||||
for child in children or []:
|
||||
crk = str(child.get("ratingKey") or "")
|
||||
if crk:
|
||||
membership.setdefault(crk, []).append(rk)
|
||||
@@ -787,9 +775,4 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
||||
for rk, col in existing.items():
|
||||
if rk not in seen:
|
||||
db.delete(col)
|
||||
# Rebuild collection_keys on every member row of this library (clear + set).
|
||||
for row in db.execute(select(PlexItem).where(PlexItem.library_id == lib.id)).scalars():
|
||||
row.collection_keys = membership.get(row.rating_key) or None
|
||||
for row in db.execute(select(PlexShow).where(PlexShow.library_id == lib.id)).scalars():
|
||||
row.collection_keys = membership.get(row.rating_key) or None
|
||||
db.flush()
|
||||
_rebuild_membership(db, lib, membership, kept)
|
||||
|
||||
+183
-30
@@ -1,4 +1,4 @@
|
||||
"""TMDB enrichment for the mirrored Plex catalog (E5 S2).
|
||||
"""TMDB enrichment for the mirrored Plex catalog.
|
||||
|
||||
A thin synchronous TMDB v3 client plus the parse/persist of the enrichment fields onto plex_items /
|
||||
plex_shows, matched by the stored canonical `guid` ("tmdb:12345", or imdb/tvdb resolved via /find).
|
||||
@@ -10,8 +10,10 @@ One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `
|
||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||
"""
|
||||
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import cast
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import func, select
|
||||
@@ -21,6 +23,8 @@ from sqlalchemy.orm import Session
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import PlexItem, PlexPerson, PlexShow
|
||||
from app.plex.cards import display_title
|
||||
from app.plex.people import people_by_credits, warm_portraits
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||
@@ -124,9 +128,17 @@ class TmdbClient:
|
||||
"""Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find."""
|
||||
return self._get(f"/find/{external_id}", {"external_source": source})
|
||||
|
||||
def countries(self) -> list[dict] | None:
|
||||
"""TMDB's country list: `[{iso_3166_1, english_name, native_name}]` (a JSON array, which
|
||||
`_get`'s dict annotation does not describe)."""
|
||||
return cast("list[dict] | None", self._get("/configuration/countries"))
|
||||
|
||||
def fetch_person(self, person_id: int) -> dict | None:
|
||||
"""One person's bio + combined credits (E5 S6), in a single append_to_response request."""
|
||||
return self._get(f"/person/{person_id}", {"append_to_response": "combined_credits"})
|
||||
"""One person's bio + combined credits + the bio's translations, in a single
|
||||
append_to_response request."""
|
||||
return self._get(
|
||||
f"/person/{person_id}", {"append_to_response": "combined_credits,translations"}
|
||||
)
|
||||
|
||||
|
||||
def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None:
|
||||
@@ -214,7 +226,7 @@ def parse_tmdb(payload: dict, media_type: str) -> dict:
|
||||
crew.sort(key=lambda c: priority.get(c["job"], 99))
|
||||
crew = crew[:12]
|
||||
|
||||
# name -> tmdb person id, for the plex_people foundation (S6 fetches bios off these ids).
|
||||
# name -> tmdb person id, for the plex_people foundation (bios are fetched off these ids).
|
||||
people: list[dict] = []
|
||||
for c in (credits.get("cast") or [])[:30]:
|
||||
if c.get("name") and c.get("id"):
|
||||
@@ -254,8 +266,12 @@ def parse_tmdb(payload: dict, media_type: str) -> dict:
|
||||
c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")
|
||||
][:8]
|
||||
or None,
|
||||
# The ISO code beside TMDB's English name: the client names the country in the reader's
|
||||
# language from the code; the name is the fallback for a code it cannot name.
|
||||
"production_countries": [
|
||||
c.get("name") for c in (payload.get("production_countries") or []) if c.get("name")
|
||||
{"code": c.get("iso_3166_1") or None, "name": c["name"]}
|
||||
for c in (payload.get("production_countries") or [])
|
||||
if c.get("name")
|
||||
]
|
||||
or None,
|
||||
"recommendations": recs or None,
|
||||
@@ -301,8 +317,8 @@ def _link_people(db: Session, links: dict[str, int]) -> None:
|
||||
|
||||
|
||||
def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the remainder —
|
||||
mirrors _enrich_people_batch). A transient TmdbError leaves the watermark NULL for retry; a
|
||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the
|
||||
remainder). A transient TmdbError leaves the watermark NULL for retry; a
|
||||
definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched
|
||||
every run. One commit at the end. Returns the number processed (matched OR definitively unmatched)."""
|
||||
movies = (
|
||||
@@ -356,13 +372,62 @@ def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||
return processed
|
||||
|
||||
|
||||
def _country(c: dict | str) -> dict:
|
||||
"""One stored production country as served: `{code, name}`. An older row stored the
|
||||
bare English name; until `backfill_country_codes` reaches it, it serves with no code (the client
|
||||
shows the name)."""
|
||||
return c if isinstance(c, dict) else {"code": None, "name": c}
|
||||
|
||||
|
||||
def backfill_country_codes(db: Session, tmdb: TmdbClient) -> int:
|
||||
"""Give the older countries (bare English names) their ISO codes, from TMDB's own
|
||||
country list — the names came from that list, so they match it exactly. One request, and only
|
||||
while such a row is left: re-fetching each title instead would take the enrichment batch days.
|
||||
A name the list does not hold keeps no code. Returns the number of rows rewritten."""
|
||||
stale = [
|
||||
row
|
||||
for model in (PlexItem, PlexShow)
|
||||
for row in db.execute(
|
||||
select(model).where(func.jsonb_typeof(model.production_countries[0]) == "string")
|
||||
).scalars()
|
||||
]
|
||||
if not stale:
|
||||
return 0
|
||||
codes = {
|
||||
c["english_name"]: c["iso_3166_1"]
|
||||
for c in tmdb.countries() or []
|
||||
if c.get("english_name") and c.get("iso_3166_1")
|
||||
}
|
||||
for row in stale:
|
||||
row.production_countries = [
|
||||
{"code": codes.get(c), "name": c} if isinstance(c, str) else c
|
||||
for c in row.production_countries or []
|
||||
]
|
||||
db.commit()
|
||||
return len(stale)
|
||||
|
||||
|
||||
def run_enrich(db: Session) -> dict:
|
||||
"""Scheduler entry point. No-op (never an error) when the TMDB key isn't configured."""
|
||||
if not sysconfig.get_str(db, "tmdb_api_key"):
|
||||
return {"enriched": 0, "skipped": "tmdb_api_key not set"}
|
||||
with TmdbClient(db) as tmdb:
|
||||
coded = backfill_country_codes(db, tmdb)
|
||||
n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch)
|
||||
return {"enriched": n}
|
||||
return {"enriched": n, **({"countries_coded": coded} if coded else {})}
|
||||
|
||||
|
||||
def run_people_enrich(db: Session) -> dict:
|
||||
"""Scheduler entry point of people_enrich: a batch of person bios (the most-credited
|
||||
first) and a batch of portraits. Each half is a no-op — never an error — without its TMDB key /
|
||||
without Plex."""
|
||||
batch = settings.people_enrich_batch
|
||||
if sysconfig.get_str(db, "tmdb_api_key"):
|
||||
with TmdbClient(db) as tmdb:
|
||||
bios = enrich_people_batch(db, tmdb, batch)
|
||||
else:
|
||||
bios = {"skipped": "tmdb_api_key not set"}
|
||||
return {"bios": bios, "portraits": warm_portraits(db, batch)}
|
||||
|
||||
|
||||
def resolve_library_guids(
|
||||
@@ -406,10 +471,11 @@ def resolve_library_guids(
|
||||
return movies, shows
|
||||
|
||||
|
||||
def resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]:
|
||||
def resolve_recs(db: Session, recs: list[dict] | None, *, lang: str) -> list[dict]:
|
||||
"""Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:<id>`, bucketed
|
||||
by media_type — see resolve_library_guids) and, if so, its rating_key — so the client can deep-link
|
||||
the ones we have and merely display the rest."""
|
||||
the ones we have and merely display the rest. One we have is named as the library names it, in
|
||||
`lang`; one we lack keeps the title TMDB gave."""
|
||||
if not recs:
|
||||
return []
|
||||
movies, shows = resolve_library_guids(db, recs)
|
||||
@@ -417,13 +483,19 @@ def resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]:
|
||||
for r in recs:
|
||||
g = f"tmdb:{r.get('tmdb_id')}"
|
||||
row = shows.get(g) if r.get("media_type") == "tv" else movies.get(g)
|
||||
named = {"title": display_title(row, lang)} if row is not None else {}
|
||||
out.append(
|
||||
{**r, "in_library": row is not None, "rating_key": row.rating_key if row else None}
|
||||
{
|
||||
**r,
|
||||
**named,
|
||||
"in_library": row is not None,
|
||||
"rating_key": row.rating_key if row else None,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def serve_block(db: Session, row) -> dict | None:
|
||||
def serve_block(db: Session, row, *, lang: str) -> dict | None:
|
||||
"""The TMDB extras for a movie/show detail response, served from the row (no live TMDB call).
|
||||
None until the row has been TMDB-enriched (the client then shows nothing extra)."""
|
||||
if row.tmdb_enriched_at is None:
|
||||
@@ -437,30 +509,58 @@ def serve_block(db: Session, row) -> dict | None:
|
||||
"keywords": row.keywords or [],
|
||||
"crew": row.crew or [],
|
||||
"studios": row.studios or [],
|
||||
"production_countries": row.production_countries or [],
|
||||
"recommendations": resolve_recs(db, row.recommendations),
|
||||
"production_countries": [_country(c) for c in row.production_countries or []],
|
||||
"recommendations": resolve_recs(db, row.recommendations, lang=lang),
|
||||
"backdrops": row.backdrops or [],
|
||||
}
|
||||
|
||||
|
||||
# --- Person enrichment (E5 S6) -------------------------------------------------------------------
|
||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once (the first person-page open)
|
||||
# and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||
# --- Person enrichment -------------------------------------------------------------------
|
||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once — by the people_enrich job, or
|
||||
# by the first person-page open if the job has not reached them yet — and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||
# endpoint resolves them to library rows by guid at serve time (like recommendations).
|
||||
|
||||
_KNOWN_FOR_MAX = 15
|
||||
# The candidate pool kept per person. Only the titles that are in the library are shown (at most
|
||||
# routes.plex.people._KNOWN_FOR_SHOWN), so the pool is deeper than the rail: a star's 15 most-voted
|
||||
# credits are often not the 15 a given library holds.
|
||||
_KNOWN_FOR_MAX = 40
|
||||
# TMDB genre ids of the shows a person visits as a guest (News, Reality, Talk). Ranked by popularity,
|
||||
# talk-show visits crowded out the films (Tom Cruise: 14 of 15 were talk shows).
|
||||
_GUEST_GENRES = {10763, 10764, 10767}
|
||||
# A credit in which the person plays themselves ("Self", "Himself - Guest", "Herself (archive footage)").
|
||||
_SELF_CREDIT = re.compile(r"^\s*(self|himself|herself|themselves)\b", re.IGNORECASE)
|
||||
# The crew jobs that make a title the person's own work (a director is known for what they directed).
|
||||
_KNOWN_FOR_CREW_JOBS = {"Director", "Creator"}
|
||||
|
||||
|
||||
def _known_for_credit(c: dict, crew: bool) -> bool:
|
||||
if crew:
|
||||
return c.get("job") in _KNOWN_FOR_CREW_JOBS
|
||||
if _SELF_CREDIT.match(c.get("character") or ""):
|
||||
return False
|
||||
return not _GUEST_GENRES.intersection(c.get("genre_ids") or [])
|
||||
|
||||
|
||||
def parse_person(payload: dict) -> dict:
|
||||
"""TMDB /person/{id} (+combined_credits) → the persisted bio fields. `known_for` = the person's
|
||||
most-popular movie/tv credits (deduped by id), top `_KNOWN_FOR_MAX`, resolved to library later."""
|
||||
credits = (payload.get("combined_credits") or {}).get("cast") or []
|
||||
most-voted movie/tv credits — roles and directed works, not guest visits as themselves — deduped,
|
||||
the top `_KNOWN_FOR_MAX`, resolved to library rows at serve time. Ranked by vote count, as TMDB's
|
||||
own "Known for" is: popularity is this week's attention, which a talk-show visit wins."""
|
||||
combined = payload.get("combined_credits") or {}
|
||||
credits = [c for c in combined.get("cast") or [] if _known_for_credit(c, crew=False)] + [
|
||||
c for c in combined.get("crew") or [] if _known_for_credit(c, crew=True)
|
||||
]
|
||||
# Dedup on (id, media_type), not the bare id: TMDB movie ids and tv ids are independent sequences
|
||||
# (id 603 can be both a movie and a show — see resolve_library_guids), so a same-id movie+show pair
|
||||
# is two distinct credits and both must survive.
|
||||
seen: set[tuple[int, str]] = set()
|
||||
known: list[dict] = []
|
||||
for c in sorted(credits, key=lambda c: c.get("popularity") or 0, reverse=True):
|
||||
ranked = sorted(
|
||||
credits,
|
||||
key=lambda c: (c.get("vote_count") or 0, c.get("popularity") or 0),
|
||||
reverse=True,
|
||||
)
|
||||
for c in ranked:
|
||||
tid, mt = c.get("id"), c.get("media_type")
|
||||
if not tid or mt not in ("movie", "tv") or (tid, mt) in seen:
|
||||
continue
|
||||
@@ -470,6 +570,7 @@ def parse_person(payload: dict) -> dict:
|
||||
break
|
||||
return {
|
||||
"biography": (payload.get("biography") or "").strip() or None,
|
||||
"biography_hu": _translated_bio(payload, "hu"),
|
||||
"birthday": payload.get("birthday") or None,
|
||||
"deathday": payload.get("deathday") or None,
|
||||
"place_of_birth": payload.get("place_of_birth") or None,
|
||||
@@ -479,8 +580,17 @@ def parse_person(payload: dict) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _translated_bio(payload: dict, lang: str) -> str | None:
|
||||
"""The `lang` biography from the appended `translations` (TMDB leaves an untranslated one blank)."""
|
||||
for tr in (payload.get("translations") or {}).get("translations") or []:
|
||||
if tr.get("iso_639_1") == lang:
|
||||
return ((tr.get("data") or {}).get("biography") or "").strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def apply_person(row: PlexPerson, parsed: dict) -> None:
|
||||
row.biography = parsed["biography"]
|
||||
row.biography_hu = parsed["biography_hu"]
|
||||
row.birthday = parsed["birthday"]
|
||||
row.deathday = parsed["deathday"]
|
||||
row.place_of_birth = parsed["place_of_birth"]
|
||||
@@ -490,11 +600,22 @@ def apply_person(row: PlexPerson, parsed: dict) -> None:
|
||||
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _fetch_person_into(tmdb: TmdbClient, person: PlexPerson) -> None:
|
||||
"""Fetch one person from TMDB and write the row (not committed). A 404 (deleted person) stamps the
|
||||
watermark with empty fields so it isn't fetched again. Raises TmdbError on a transient failure,
|
||||
leaving the row untouched and `tmdb_enriched_at` NULL for a retry."""
|
||||
payload = tmdb.fetch_person(cast(int, person.tmdb_person_id))
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
else:
|
||||
apply_person(person, parse_person(payload))
|
||||
|
||||
|
||||
def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
"""Lazily fetch + persist one person's TMDB bio (first person-page open), committing the write so it
|
||||
survives regardless of the read path. No-op (False) without a key or a `tmdb_person_id`; a transient
|
||||
error leaves `tmdb_enriched_at` NULL so the next open retries. A 404 (deleted person) stamps the
|
||||
watermark with empty fields so we don't refetch it every open."""
|
||||
"""Lazily fetch + persist one person's TMDB bio (a person page opened before the background job
|
||||
reached them), committing the write so it survives regardless of the read path. No-op (False)
|
||||
without a key or a `tmdb_person_id`; a transient error leaves `tmdb_enriched_at` NULL so the next
|
||||
open retries."""
|
||||
if not person.tmdb_person_id or not sysconfig.get_str(db, "tmdb_api_key"):
|
||||
return False
|
||||
try:
|
||||
@@ -502,7 +623,7 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
# longer blocks a serve; still cap the outbound TMDB call with a short timeout so a hanging TMDB
|
||||
# doesn't tie up a threadpool worker for the default 30s.
|
||||
with TmdbClient(db, timeout=8.0) as tmdb:
|
||||
payload = tmdb.fetch_person(person.tmdb_person_id)
|
||||
_fetch_person_into(tmdb, person)
|
||||
except TmdbError as exc:
|
||||
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
||||
# front of the user right now shows an empty biography with nothing to say why.
|
||||
@@ -515,9 +636,41 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
),
|
||||
)
|
||||
return False
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
else:
|
||||
apply_person(person, parse_person(payload))
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
# Lookups failing this many people in a row end a people batch: TMDB itself is degraded, and hammering
|
||||
# it helps nobody. One person failing alone is skipped, so an id TMDB keeps refusing never stalls the rest.
|
||||
_PEOPLE_FAILS_IN_A_ROW = 3
|
||||
|
||||
|
||||
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
||||
"""Fetch up to `limit` linked, not-yet-enriched people, the most-credited first, so a
|
||||
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
||||
lands: a page open during the run already sees it. A failed lookup is skipped (left unstamped for the
|
||||
next run); _PEOPLE_FAILS_IN_A_ROW of them in a row end the batch."""
|
||||
ids = people_by_credits(db, "bio", limit)
|
||||
done = failed = in_a_row = 0
|
||||
for pid in ids:
|
||||
if in_a_row >= _PEOPLE_FAILS_IN_A_ROW:
|
||||
break
|
||||
person = db.get(PlexPerson, pid)
|
||||
# A page open may have enriched this person since the select: skip, don't fetch twice.
|
||||
if person is None or person.tmdb_enriched_at is not None:
|
||||
continue
|
||||
try:
|
||||
_fetch_person_into(tmdb, person)
|
||||
except TmdbError as exc:
|
||||
db.rollback()
|
||||
failed, in_a_row = failed + 1, in_a_row + 1
|
||||
log.warning(
|
||||
"a TMDB person lookup in the batch failed",
|
||||
extra=logs.event(
|
||||
"plex.tmdb.people_batch_failed", person_id=pid, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
continue
|
||||
db.commit()
|
||||
done, in_a_row = done + 1, 0
|
||||
return {"enriched": done, **({"failed": failed} if failed else {})}
|
||||
|
||||
@@ -11,7 +11,7 @@ already pages) and upsert them into the owner's `plex_states`. Plex wins on over
|
||||
exist only in Siftlode are left untouched (union, Plex-authoritative on the intersection).
|
||||
|
||||
Phases B (Siftlode→Plex push via scrobble/timeline) and C (incremental Plex→Siftlode via the watch
|
||||
history + onDeck) build on this and land in later ships. As of S2 Plex is the source of truth: the
|
||||
history + onDeck) build on this and land in later ships. Plex is now the source of truth: the
|
||||
player pushes the resume position back to Plex on the periodic checkpoint (not just the final flush),
|
||||
and the incremental pull resolves a genuine divergence to Plex's state (see `_pull_apply`) rather than
|
||||
running a last-write-wins comparison.
|
||||
@@ -29,7 +29,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.db import SessionLocal
|
||||
from app.models import PlexItem, PlexLink, PlexState
|
||||
from app.models import PlexItem, PlexLibrary, PlexLink, PlexState
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.sync import (
|
||||
_TYPE_EPISODE,
|
||||
@@ -72,13 +72,27 @@ def _rk_to_id(db: Session) -> dict[str, int]:
|
||||
|
||||
def _scan_plex_states(
|
||||
db: Session, plex: PlexClient
|
||||
) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int]:
|
||||
) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int, set[str], set[str]]:
|
||||
"""Scan the enabled movie/show sections and return ``{item_id: (status, pos, watched_at,
|
||||
prog_at)}`` for every mirrored leaf Plex holds a watch signal for, plus the scanned-leaf count.
|
||||
Shared by the one-time import and the full reconcile (both need Plex's whole current picture)."""
|
||||
prog_at)}`` for every mirrored leaf Plex holds a watch signal for, the scanned-leaf count, the
|
||||
keys of the sections SETTLED — walked to the server's total (or said empty by a server we mirror
|
||||
nothing from) — and the keys of every section walked.
|
||||
Shared by the one-time import and the full reconcile (both need Plex's whole current picture).
|
||||
The settled keys matter to the reconcile alone: it reads "absent from the scan" as "Plex
|
||||
un-watched it", which a walk cut short by a Plex hiccup does not mean (see `_paginate`). Settled
|
||||
is decided PER SECTION: one section that never settles (a library emptied on Plex while its
|
||||
mirrored items are kept) must not stop un-watches syncing in all the others."""
|
||||
item_id_by_rk = _rk_to_id(db)
|
||||
wanted = _enabled_section_keys(db)
|
||||
# The sections we mirror something from: a total of 0 from one of those contradicts the mirror.
|
||||
mirrored = set(
|
||||
db.execute(
|
||||
select(PlexLibrary.plex_key).join(PlexItem, PlexItem.library_id == PlexLibrary.id)
|
||||
).scalars()
|
||||
)
|
||||
scanned = 0
|
||||
settled: set[str] = set()
|
||||
walked: set[str] = set()
|
||||
out: dict[int, tuple[str, int, datetime | None, datetime | None]] = {}
|
||||
for s in plex.sections():
|
||||
stype = s.get("type")
|
||||
@@ -88,7 +102,10 @@ def _scan_plex_states(
|
||||
if wanted is not None and key not in wanted:
|
||||
continue
|
||||
item_type = _TYPE_MOVIE if stype == "movie" else _TYPE_EPISODE
|
||||
for meta in _paginate(plex, key, item_type):
|
||||
walked.add(key)
|
||||
progress: dict = {}
|
||||
before = scanned
|
||||
for meta in _paginate(plex, key, item_type, progress):
|
||||
scanned += 1
|
||||
item_id = item_id_by_rk.get(str(meta.get("ratingKey") or ""))
|
||||
if item_id is None:
|
||||
@@ -98,7 +115,15 @@ def _scan_plex_states(
|
||||
)
|
||||
if res is not None:
|
||||
out[item_id] = res
|
||||
return out, scanned
|
||||
# A section the server SAYS is empty (total 0, nothing listed) settles too. Believed only for
|
||||
# a section we mirror nothing from: one whose items we hold answering 0 is a Plex that lost
|
||||
# the library for a moment, not one that un-watched all of it — so it is left
|
||||
# unsettled, and its rows are not cleared.
|
||||
listed = scanned > before
|
||||
said_empty = progress.get("total") == 0 and not listed and key not in mirrored
|
||||
if (progress.get("complete") and listed) or said_empty:
|
||||
settled.add(key)
|
||||
return out, scanned, settled, walked
|
||||
|
||||
|
||||
def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
||||
@@ -111,7 +136,8 @@ def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
||||
stats = {"watched": 0, "in_progress": 0, "scanned": 0}
|
||||
try:
|
||||
with PlexClient(db) as plex:
|
||||
scanned_states, stats["scanned"] = _scan_plex_states(db, plex)
|
||||
# A partial scan is still safe here: the import only ever ADDS what Plex reports.
|
||||
scanned_states, stats["scanned"], _settled, _walked = _scan_plex_states(db, plex)
|
||||
# One row per item (keyed by item_id → inherently de-duped) then bulk-UPSERT. An upsert is
|
||||
# idempotent AND concurrency-safe: a repeated/overlapping enable (the toggle firing twice
|
||||
# while the multi-second import runs) can't raise a duplicate-key error — it re-updates.
|
||||
@@ -376,7 +402,7 @@ def _pull_apply(
|
||||
if st is not None and not st.synced_to_plex:
|
||||
st.synced_to_plex = True # already matches Plex → just settle the flag, no churn
|
||||
return None
|
||||
# Plex-authoritative (S2): on a genuine divergence for an item Plex knows about, Plex wins — no
|
||||
# Plex-authoritative: on a genuine divergence for an item Plex knows about, Plex wins — no
|
||||
# last-write-wins tug-of-war. This is safe because prompt write-back (playback.py pushes the resume
|
||||
# position to Plex on the ~10s checkpoint, not just on the final flush) keeps Plex current, so a
|
||||
# local edit that's "newer" than Plex has almost always already been pushed and matched above by
|
||||
@@ -590,6 +616,27 @@ def run_plex_watch_sync(db: Session) -> dict:
|
||||
return total
|
||||
|
||||
|
||||
def _clear_unwatched(db: Session, user_id: int, plex_states: dict, settled: set[str]) -> int:
|
||||
"""Clear the user's rows Plex no longer has a record for — the mirrored ones (Plex un-watched
|
||||
them); hidden and dirty rows stay (a dirty one is re-pushed by `_repush_dirty`). Only rows of the
|
||||
sections the scan SETTLED: a walk a Plex hiccup cut short leaves out rows it merely failed to
|
||||
reach, and clearing those un-watched them here until the next complete run; an item of
|
||||
a library the allowlist leaves out is absent from every scan, and was cleared on every run."""
|
||||
n = 0
|
||||
rows = db.execute(
|
||||
select(PlexState)
|
||||
.join(PlexItem, PlexItem.id == PlexState.item_id)
|
||||
.join(PlexLibrary, PlexLibrary.id == PlexItem.library_id)
|
||||
.where(PlexState.user_id == user_id, PlexLibrary.plex_key.in_(settled))
|
||||
)
|
||||
for st in rows.scalars().all():
|
||||
if st.item_id in plex_states or st.status == "hidden" or not st.synced_to_plex:
|
||||
continue
|
||||
db.delete(st)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def run_plex_watch_reconcile(db: Session) -> dict:
|
||||
"""Scheduler job (default daily) — the authoritative full reconcile. Rescans every section and,
|
||||
per item, uses `synced_to_plex` to settle what the incremental feed can't:
|
||||
@@ -608,7 +655,7 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
||||
try:
|
||||
with PlexClient(db) as plex:
|
||||
_resolve_account_id(db, plex, link)
|
||||
plex_states, scanned = _scan_plex_states(db, plex)
|
||||
plex_states, scanned, settled, walked = _scan_plex_states(db, plex)
|
||||
total["scanned"] += scanned
|
||||
|
||||
for item_id, (status, pos, watched_at, prog_at) in plex_states.items():
|
||||
@@ -616,18 +663,19 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
||||
if ch:
|
||||
total["applied"] += 1
|
||||
|
||||
# Siftlode rows Plex no longer has a record for: clear the ones that were mirrored
|
||||
# (Plex un-watched them); leave hidden + dirty (dirty gets re-pushed just below).
|
||||
for st in (
|
||||
db.execute(select(PlexState).where(PlexState.user_id == link.user_id))
|
||||
.scalars()
|
||||
.all()
|
||||
):
|
||||
if st.item_id in plex_states or st.status == "hidden":
|
||||
continue
|
||||
if st.synced_to_plex:
|
||||
db.delete(st)
|
||||
total["cleared"] += 1
|
||||
total["cleared"] += _clear_unwatched(db, link.user_id, plex_states, settled)
|
||||
if unsettled := sorted(walked - settled):
|
||||
total["incomplete"] = total.get("incomplete", 0) + 1
|
||||
log.warning(
|
||||
"Plex sections %s were cut short or answered empty; their rows are not cleared"
|
||||
" this run",
|
||||
", ".join(unsettled),
|
||||
extra=logs.event(
|
||||
"plex.watch.reconcile_incomplete",
|
||||
user_id=link.user_id,
|
||||
sections=unsettled,
|
||||
),
|
||||
)
|
||||
|
||||
total["repushed"] += _repush_dirty(db, plex, link)
|
||||
link.last_watch_sync_at = datetime.now(timezone.utc)
|
||||
|
||||
+17
-2
@@ -8,7 +8,7 @@ its own thread/context) never cross wires. When no sink is bound (e.g. a manual
|
||||
"""
|
||||
|
||||
import contextvars
|
||||
from typing import Callable, Optional
|
||||
from typing import Callable, Literal, Optional
|
||||
|
||||
Sink = Callable[[int, Optional[int], Optional[str]], None]
|
||||
|
||||
@@ -24,6 +24,19 @@ def report(current: int, total: int | None = None, phase: str | None = None) ->
|
||||
sink(current, total, phase)
|
||||
|
||||
|
||||
class JobFailed(RuntimeError):
|
||||
"""A job's own failure, said so the reader's language can name it: the client reads
|
||||
`scheduler.failure.<reason>` and interpolates `params` (keep them to values the reader may see —
|
||||
numbers, ids, short codes). The message is the English line the log and `last_error` keep. An
|
||||
upstream's own error (a timeout, a refused connection) is not one of these: it stays as raised,
|
||||
being diagnostic."""
|
||||
|
||||
def __init__(self, reason: str, message: str, **params: object):
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
self.params = params
|
||||
|
||||
|
||||
class bind:
|
||||
"""Context manager that binds a progress sink for the duration of a job run."""
|
||||
|
||||
@@ -35,7 +48,9 @@ class bind:
|
||||
self._token = _sink.set(self._sink)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> bool:
|
||||
# Literal[False], not bool: this never swallows the job's exception, and saying so is what lets
|
||||
# a type checker know a name assigned inside `with bind(...)` is bound after it.
|
||||
def __exit__(self, *exc) -> Literal[False]:
|
||||
if self._token is not None:
|
||||
_sink.reset(self._token)
|
||||
return False
|
||||
|
||||
+18
-2
@@ -7,11 +7,13 @@ always be allowed; expensive backfill must yield to the remaining budget.
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, cast
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
@@ -96,6 +98,20 @@ def pacific_today():
|
||||
return datetime.now(_PACIFIC).date()
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Delete quota-history rows older than `retention_days` (<= 0 keeps them forever) and return how
|
||||
many went. Flushes only; the cleanup job commits. Today's rows are never in reach: the per-user
|
||||
search cap counts them, and the floor of two days outlasts any Pacific day (25 h at most)."""
|
||||
if retention_days <= 0:
|
||||
return 0
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(2, retention_days))
|
||||
# A DELETE answers with a CursorResult; `execute` is typed as the general Result, without rowcount.
|
||||
result = cast(
|
||||
CursorResult[Any], db.execute(delete(QuotaEvent).where(QuotaEvent.created_at < cutoff))
|
||||
)
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def pacific_day_start_utc() -> datetime:
|
||||
"""UTC instant of the current Pacific day's midnight — aligns 'today' with the budget reset."""
|
||||
start = datetime.now(_PACIFIC).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
"""In-process WebSocket connection registry for live server→client push.
|
||||
|
||||
Generic groundwork (notification module phase 2 is the first user): connections are kept in
|
||||
this process's memory, keyed by user_id, so a request handler can push an event to all of a
|
||||
user's open tabs/devices. Single-uvicorn-process assumption — if we ever run multiple workers,
|
||||
replace the in-memory map with a Redis pub/sub fan-out behind the same `push()` interface (same
|
||||
note as app.ratelimit).
|
||||
|
||||
`push()` is intentionally SYNC-callable: ordinary (sync) route handlers run in a threadpool, so
|
||||
they schedule the actual send onto the WebSocket event loop via run_coroutine_threadsafe. The
|
||||
connect/disconnect/_push coroutines all run on that one loop, so the connection map is only ever
|
||||
mutated from a single thread.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections import defaultdict
|
||||
|
||||
from starlette.websockets import WebSocket
|
||||
from app import logs
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
def __init__(self) -> None:
|
||||
self._conns: dict[int, set[WebSocket]] = defaultdict(set)
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
async def connect(self, user_id: int, ws: WebSocket) -> None:
|
||||
self._loop = asyncio.get_running_loop()
|
||||
await ws.accept()
|
||||
self._conns[user_id].add(ws)
|
||||
|
||||
async def disconnect(self, user_id: int, ws: WebSocket) -> None:
|
||||
conns = self._conns.get(user_id)
|
||||
if conns:
|
||||
conns.discard(ws)
|
||||
if not conns:
|
||||
self._conns.pop(user_id, None)
|
||||
|
||||
async def _push(self, user_id: int, payload: dict) -> None:
|
||||
for ws in list(self._conns.get(user_id, ())):
|
||||
try:
|
||||
await ws.send_json(payload)
|
||||
except Exception as exc: # noqa: BLE001 — one dead socket must not stop the others
|
||||
# Stale socket; it'll be cleaned up by its own disconnect handler. But this is
|
||||
# also where a live update goes missing: the UI simply never changes, and there
|
||||
# is no request behind it whose failure anyone could look up.
|
||||
log.debug(
|
||||
"a live update could not be pushed to a socket",
|
||||
extra=logs.event(
|
||||
"realtime.push_failed", user_id=user_id, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
|
||||
def push(self, user_id: int, payload: dict) -> None:
|
||||
"""Schedule a push onto the WebSocket event loop. Safe to call from a sync route
|
||||
handler (threadpool). No-op if no WebSocket has ever connected (no loop captured)."""
|
||||
loop = self._loop
|
||||
if loop is None:
|
||||
return
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(self._push(user_id, payload), loop)
|
||||
except Exception: # noqa: BLE001 — a closed loop or socket must not fail the caller
|
||||
log.debug("ws push failed", exc_info=True)
|
||||
|
||||
|
||||
manager = ConnectionManager()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user