Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8247c0dde | ||
|
|
722c37a6d0 | ||
|
|
5ba5679239 | ||
|
|
926e5e9899 | ||
|
|
9251918e08 | ||
|
|
2b6d968833 | ||
|
|
95e6b2c3d9 | ||
|
|
80c55fbf1b | ||
|
|
011f269152 | ||
|
|
dcb244dd04 | ||
|
|
08a9f4e255 | ||
|
|
1673134c36 | ||
|
|
27427eef32 | ||
|
|
a248ca6c13 | ||
|
|
620f84dba4 | ||
|
|
2efa7d5a75 | ||
|
|
827df7494c | ||
|
|
86712b6b0e | ||
|
|
90567718b2 | ||
|
|
c31d7f9d65 | ||
|
|
db56f25387 | ||
|
|
4f58d15d3f | ||
|
|
2c66ce1cf4 | ||
|
|
ce45445d17 | ||
|
|
383d394e32 | ||
|
|
06e8291896 | ||
|
|
db6b60029f | ||
|
|
02134799cc | ||
|
|
c8bc843517 | ||
|
|
759a7536b9 | ||
|
|
b2d34e6e6c | ||
|
|
c335d54929 | ||
|
|
7188ad7d71 | ||
|
|
ebbae6794c | ||
|
|
c9baf68b16 | ||
|
|
5bb101e493 | ||
|
|
54a6bbf07f | ||
|
|
8b2260c9a8 | ||
|
|
815083bcc4 | ||
|
|
96509fe356 | ||
|
|
132fb56428 | ||
|
|
f6bce23a94 | ||
|
|
fb8fb980e5 | ||
|
|
6707550513 | ||
|
|
078194731b | ||
|
|
1c4f0042d2 | ||
|
|
d22c64f4dd | ||
|
|
85aa917b38 | ||
|
|
1a1665b8ec | ||
|
|
bf99444ded | ||
|
|
07e0990ecf | ||
|
|
e2ed2bf874 | ||
|
|
b9eb8458b3 | ||
|
|
891507dc75 | ||
|
|
e6894ac0fc | ||
|
|
663a7e8012 | ||
|
|
93192dadd8 | ||
|
|
5b48018e92 | ||
|
|
8312c6d592 | ||
|
|
e1d325b942 | ||
|
|
977ba244fa | ||
|
|
89d18b46a6 | ||
|
|
c27cfdcb9c | ||
|
|
a867145bcb | ||
|
|
4d4eea2c54 | ||
|
|
c489b60c8b | ||
|
|
b4bf71ea64 | ||
|
|
72df11f2c0 | ||
|
|
0a16bc40ae | ||
|
|
520b0afb90 | ||
|
|
e301632765 | ||
|
|
50157cc2fd | ||
|
|
f221c1962a | ||
|
|
d466ae915b | ||
|
|
14598ec320 | ||
|
|
b16cf7668c | ||
|
|
3438aeb695 | ||
|
|
c003a48244 | ||
|
|
8a2334e198 | ||
|
|
1f010328df | ||
|
|
77638f7a0b | ||
|
|
2cff4ad954 | ||
|
|
8776586ca3 | ||
|
|
92a17aba83 | ||
|
|
fc53bc03bc | ||
|
|
6df257d7df | ||
|
|
5572499b56 | ||
|
|
76738f8ea9 | ||
|
|
568bf93510 | ||
|
|
942154cf49 | ||
|
|
3eb87e7108 | ||
|
|
851b97b09d | ||
|
|
ee94d1949a | ||
|
|
4f076b5d45 | ||
|
|
ece74a42ab | ||
|
|
600d5c39da | ||
|
|
4c7f73af9d | ||
|
|
5f42ff61c8 | ||
|
|
6dd82ef5e2 | ||
|
|
75f4efde98 | ||
|
|
c0059a0c4b | ||
|
|
a29cadf03e | ||
|
|
83886494ce | ||
|
|
4eaa55385f | ||
|
|
7541b1b016 | ||
|
|
24ad17fc7d | ||
|
|
48bd038c5f | ||
|
|
0fce732497 | ||
|
|
7e94da72b9 | ||
|
|
fadb4f17a9 | ||
|
|
57a81c5643 | ||
|
|
dae0b80f6a | ||
|
|
75093458c9 | ||
|
|
5db6dc8ee6 | ||
|
|
45a9ceae25 | ||
|
|
5249c41edc | ||
|
|
c3664826fe | ||
|
|
c9b8fe15a3 | ||
|
|
1044eaf8a9 | ||
|
|
ca949d22de | ||
|
|
20178f0ec4 | ||
|
|
49a9319f84 | ||
|
|
70d56ab939 | ||
|
|
3865689c26 | ||
|
|
c32429ba69 | ||
|
|
a504550196 | ||
|
|
5095047c81 | ||
|
|
4e70f168b1 | ||
|
|
7f17f488ab | ||
|
|
d1704a5519 | ||
|
|
121ebeebc1 | ||
|
|
ee5290ba01 | ||
|
|
dd6a666ad1 | ||
|
|
392099770a | ||
|
|
2aa3ac6e85 | ||
|
|
891dc628f3 | ||
|
|
0a35f2a3b1 | ||
|
|
d25a132495 | ||
|
|
61d3a1a189 | ||
|
|
d482a5e8e2 | ||
|
|
2064373ac2 | ||
|
|
0cda5b6dd4 | ||
|
|
f3c94641a7 | ||
|
|
59b27467ab | ||
|
|
02feaa9a1c | ||
|
|
c1058f1d5a | ||
|
|
23f16b69df | ||
|
|
c813814c3c | ||
|
|
ed6facb703 | ||
|
|
d44d8d6dda | ||
|
|
b20e9a3947 | ||
|
|
6073a006b8 | ||
|
|
dba4758c59 | ||
|
|
68cfc7e25d | ||
|
|
00886d92fe | ||
|
|
e7d2eb8cf6 | ||
|
|
e886890d6e | ||
|
|
5561d9a379 | ||
|
|
2768ab5bdd |
+5
-5
@@ -73,13 +73,13 @@ SMTP_FROM=
|
|||||||
# of them and false on the rest, to avoid double quota use and write races.
|
# of them and false on the rest, to avoid double quota use and write races.
|
||||||
SCHEDULER_ENABLED=true
|
SCHEDULER_ENABLED=true
|
||||||
|
|
||||||
# --- Download center ---
|
# --- Files library & downloads ---
|
||||||
# The Download Center adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small
|
# Downloading adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small
|
||||||
# `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose.
|
# `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose.
|
||||||
# Downloaded media defaults to a Docker-managed named volume. Set DOWNLOAD_HOST_PATH to a host
|
# The Files library defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host
|
||||||
# directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there.
|
# directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there.
|
||||||
# The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000 <dir>.
|
# The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000 <dir>.
|
||||||
# DOWNLOAD_HOST_PATH=/mnt/media/youtube
|
# FILES_HOST_PATH=/mnt/media/youtube
|
||||||
|
|
||||||
# --- Plex integration (optional) ---
|
# --- Plex integration (optional) ---
|
||||||
# The optional Plex module browses/plays your Plex library, playing the LOCAL physical file
|
# The optional Plex module browses/plays your Plex library, playing the LOCAL physical file
|
||||||
@@ -89,6 +89,6 @@ SCHEDULER_ENABLED=true
|
|||||||
# to /plex-media inside the container (the local side of the admin "path map", whose default is
|
# to /plex-media inside the container (the local side of the admin "path map", whose default is
|
||||||
# /data=/plex-media). Leave unset if you don't use Plex.
|
# /data=/plex-media). Leave unset if you don't use Plex.
|
||||||
# PLEX_MEDIA_HOST_PATH=/mnt/media
|
# PLEX_MEDIA_HOST_PATH=/mnt/media
|
||||||
# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to DOWNLOAD_ROOT; point it at
|
# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to FILES_ROOT; point it at
|
||||||
# fast local storage if your download volume is slow.
|
# fast local storage if your download volume is slow.
|
||||||
# PLEX_HLS_DIR=/var/tmp/plex-hls
|
# PLEX_HLS_DIR=/var/tmp/plex-hls
|
||||||
|
|||||||
+2
-2
@@ -29,9 +29,9 @@ frontend/e2e/.auth/
|
|||||||
*.sql.gz
|
*.sql.gz
|
||||||
backups/
|
backups/
|
||||||
|
|
||||||
# Download center: locally downloaded media (the DOWNLOAD_ROOT bind mount at the repo root).
|
# Download center: locally downloaded media (the FILES_ROOT bind mount at the repo root).
|
||||||
# Anchored with a leading slash so it does NOT also ignore the backend/app/downloads package.
|
# Anchored with a leading slash so it does NOT also ignore the backend/app/downloads package.
|
||||||
/downloads/
|
/files/
|
||||||
|
|
||||||
# OS / editor
|
# OS / editor
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|||||||
+101
@@ -7,6 +7,107 @@ edit the release notes and re-run the generator.
|
|||||||
|
|
||||||
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
||||||
|
|
||||||
|
## 0.107.2 — 2026-10-17
|
||||||
|
|
||||||
|
### New
|
||||||
|
|
||||||
|
- Development: the gate script can list its checks (scripts/check.sh --list) and run just one (--only), and the backend tests run on every CPU core, each worker with its own database.
|
||||||
|
|
||||||
|
## 0.107.1 — 2026-10-16
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Downloads: a failed download's explanation now says to press Retry, instead of pointing to a download list that no longer exists.
|
||||||
|
|
||||||
|
## 0.107.0 — 2026-10-15
|
||||||
|
|
||||||
|
### New
|
||||||
|
|
||||||
|
- API tokens: Settings › API tokens gives one machine of yours a key to the Files folders you pick — or, in one click, to a Hub folder for swapping files with it. Signing out your other sessions or changing your password revokes every token.
|
||||||
|
- Share by link: a link can now open a whole folder of yours — its page lists what is in it and plays a video above the list.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Uploads: a finished upload is announced once, and shows in Files at once.
|
||||||
|
|
||||||
|
## 0.106.2 — 2026-10-14
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Pictures: a thumbnail, poster or channel picture that fails to load is retried, and meanwhile the card shows its empty frame instead of a broken-image mark.
|
||||||
|
|
||||||
|
## 0.106.1 — 2026-10-14
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Plex player: clicking outside the Settings panel closes the panel again instead of closing the player.
|
||||||
|
|
||||||
|
## 0.106.0 — 2026-10-13
|
||||||
|
|
||||||
|
### New
|
||||||
|
|
||||||
|
- Files: one library for everything you keep — downloads, clips and your own uploads of any file — in a folder per site plus folders of your own, each keeping its files as long as you choose. Documents, pictures and text open in a viewer.
|
||||||
|
- Downloads: paste a link anywhere on a page to download it, and the tray shows what is running and what is done. The Downloads page is gone: your files are in Files, and your download settings are under Settings › Downloads.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Self-hosting: the library is now mounted at /files (FILES_HOST_PATH). Old settings keep working; if you used the default volume, copy it over once — see the upgrade notes.
|
||||||
|
|
||||||
|
## 0.105.0 — 2026-10-10
|
||||||
|
|
||||||
|
### New
|
||||||
|
|
||||||
|
- Clips: mark a range of what you're watching — a YouTube video or a Plex item — set its ends to the tenth, crop it to a frame drawn on the picture, and cut it straight into your downloads.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Downloads: a download, a clip or an editor clip that doesn't fit in the storage you have left says so, and is never stored over your cap; YouTube downloads reach their format's height again.
|
||||||
|
- Player: focus rings, the chosen track or tab, sliders and the progress bar's head are amber on the dark player in every theme, so they show in a light theme too.
|
||||||
|
|
||||||
|
## 0.104.0 — 2026-10-10
|
||||||
|
|
||||||
|
### New
|
||||||
|
|
||||||
|
- YouTube: a card's menu filters the feed to its channel and back, and the player gives the video more room — its side panel shuts on demand.
|
||||||
|
- Plex: a card's picture plays it and its title opens its page; a show card puts on the episode it would continue or start, and an episode links up to its show.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Plex: summaries written as HTML read as clean text, and a Surprise draw stays put until you reshuffle. A video marked watched in the player stays out of an Unwatched feed. Plus smaller keyboard and focus fixes.
|
||||||
|
|
||||||
|
## 0.103.0 — 2026-10-08
|
||||||
|
|
||||||
|
### New
|
||||||
|
|
||||||
|
- Plex: “Add to collection…” in a movie's or show's menu ticks the title into or out of your collections, or names a new one with it inside — one step instead of a trip to the Collections page.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Plex player: subtitles rise above the control bar while it shows and slide back when it hides, so the bar never covers them — on an iPad too.
|
||||||
|
- iPad: the header search fits the touch screen, tooltips no longer stick, the Plex player page no longer scrolls or shows a blue band, and the fullscreen button stays hidden where the browser cannot really go fullscreen.
|
||||||
|
- The playlist, collection and tag menus say when their list is loading or failed to load (with a retry) instead of showing it as empty, and work from the keyboard. Plus smaller fixes in the YouTube player's title line.
|
||||||
|
|
||||||
|
## 0.102.0 — 2026-10-08
|
||||||
|
|
||||||
|
### New
|
||||||
|
|
||||||
|
- The library and shared-link players answer the keyboard's and the system's media keys, and resume where you stopped.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Plex player: the browser Back closes the panels over the video one at a time, and only the credits at the very end move on to the next episode. Plus smaller fixes around resume points and stopping a stream.
|
||||||
|
|
||||||
|
## 0.101.0 — 2026-10-06
|
||||||
|
|
||||||
|
### New
|
||||||
|
|
||||||
|
- A shared link plays on any phone, iPhone and iPad included: downloads, clips and uploads are kept in a format every phone plays, and an upload in another format is converted after it arrives.
|
||||||
|
- A shared link opens much faster: its page no longer loads the whole app.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- The YouTube player keeps its size on a short window and when maximised — before, it could shrink to a thin strip. Plus smaller fixes around uploads and clips.
|
||||||
|
|
||||||
## 0.100.0 — 2026-10-05
|
## 0.100.0 — 2026-10-05
|
||||||
|
|
||||||
### New
|
### New
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ be developed that way. This file is the rules of the road: read it before you to
|
|||||||
loaded into every agent session, so keep it **short and true**. A rule here that has gone stale is
|
loaded into every agent session, so keep it **short and true**. A rule here that has gone stale is
|
||||||
worse than no rule — if you change how the code works, change this file in the same commit.
|
worse than no rule — if you change how the code works, change this file in the same commit.
|
||||||
|
|
||||||
Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a download center). It is a
|
Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a files library with downloads). It is a
|
||||||
**modular monolith**: one Postgres, one FastAPI process (uvicorn + APScheduler), one worker process
|
**modular monolith**: one Postgres, one FastAPI process (uvicorn + APScheduler), one worker process
|
||||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 19 (with the React Compiler) + react-query frontend, with a
|
(yt-dlp/ffmpeg), one PO-token sidecar. React 19 (with the React Compiler) + react-query frontend, with a
|
||||||
router-free navigation provider.
|
router-free navigation provider.
|
||||||
@@ -62,10 +62,11 @@ how it gets built. Concretely:
|
|||||||
## Architecture shape (don't fight it)
|
## Architecture shape (don't fight it)
|
||||||
|
|
||||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`,
|
||||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
Frontend: `main.tsx` provider tree → `App.tsx` shell (a shared `/watch` link boots from its own
|
||||||
|
entry, `watch.html` → `watchMain.tsx`, held app-free by a contract test) → `moduleRegistry` route-chunks → module
|
||||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||||
its menus, `Modal`/`Overlay`, `Button`/`form`, `DataTable`/`VirtualGrid`, `QueryState`, …). That
|
its menus, `Modal`/`Overlay`, `Button`/`form`, `DataTable`/`VirtualGrid`, `QueryState`, …). That
|
||||||
boundary is one-way and **lint-enforced**, not just described: a platform component may not import a
|
boundary is one-way and **lint-enforced**, not just described: a platform component may not import a
|
||||||
@@ -101,6 +102,7 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
|||||||
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
||||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||||
|
| FE player on a native media element | `lib/mediaSession` (`useMediaSession`) + `lib/mediaKeys` + `lib/resume` (`useResume`), the element held as STATE | a private `navigator.mediaSession` / `keydown` / position-saving effect per player |
|
||||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||||
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
||||||
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
||||||
@@ -112,7 +114,7 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
|||||||
Not repeated here, because a lint rule refuses the wrong one and its message names the right one:
|
Not repeated here, because a lint rule refuses the wrong one and its message names the right one:
|
||||||
buttons (`ui/Button`, `IconButton`, `Pressable`, `Chip`, `DragHandle`), text fields and native
|
buttons (`ui/Button`, `IconButton`, `Pressable`, `Chip`, `DragHandle`), text fields and native
|
||||||
controls (`ui/Input`, `Checkbox`, `Slider`, …), glass surfaces (`SURFACE.*`), a native `<select>`
|
controls (`ui/Input`, `Checkbox`, `Slider`, …), glass surfaces (`SURFACE.*`), a native `<select>`
|
||||||
(`SegmentedToggle` / `FilterSelect`), and network calls (`req` / `uploadReq` / `beacon`, `lib/http.ts`).
|
(`SegmentedToggle` / `FilterSelect`), a picture (`ui/RemoteImage`, never a bare `<img>`), and network calls (`req` / `uploadReq` / `beacon`, `lib/http.ts`).
|
||||||
|
|
||||||
## House rules that bite (learned from real bugs)
|
## House rules that bite (learned from real bugs)
|
||||||
|
|
||||||
|
|||||||
+10
-4
@@ -39,13 +39,13 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Create the app user BEFORE the copies so files land already-owned via COPY --chown (a trailing
|
# Create the app user BEFORE the copies so files land already-owned via COPY --chown (a trailing
|
||||||
# `chown -R /app` would rewrite every file into a duplicate layer). /downloads must be appuser-owned
|
# `chown -R /app` would rewrite every file into a duplicate layer). /files must be appuser-owned
|
||||||
# so a fresh named volume mounted there inherits that ownership (Docker copies the image dir's
|
# so a fresh named volume mounted there inherits that ownership (Docker copies the image dir's
|
||||||
# ownership into a new empty volume) — the worker/API can then write without a manual chown. A bind
|
# ownership into a new empty volume) — the worker/API can then write without a manual chown. A bind
|
||||||
# mount to a host path instead needs that path writable by this uid (see docs/self-hosting.md).
|
# mount to a host path instead needs that path writable by this uid (see docs/self-hosting.md).
|
||||||
RUN adduser --disabled-password --gecos "" appuser \
|
RUN adduser --disabled-password --gecos "" appuser \
|
||||||
&& mkdir -p /downloads \
|
&& mkdir -p /files \
|
||||||
&& chown appuser /app /downloads
|
&& chown appuser /app /files
|
||||||
|
|
||||||
# ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode.
|
# ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode.
|
||||||
# intel-media-va-driver (iHD): the VA driver ffmpeg's h264_vaapi needs — libva2 ships in the base but
|
# intel-media-va-driver (iHD): the VA driver ffmpeg's h264_vaapi needs — libva2 ships in the base but
|
||||||
@@ -55,8 +55,14 @@ RUN adduser --disabled-password --gecos "" appuser \
|
|||||||
# token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
# token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
||||||
# `apt-get upgrade`: the base image is a snapshot, and the Debian security fixes released since it
|
# `apt-get upgrade`: the base image is a snapshot, and the Debian security fixes released since it
|
||||||
# was cut are only in the archive — the first Trivy scan found one (pcre2) fixed upstream but not in it.
|
# was cut are only in the archive — the first Trivy scan found one (pcre2) fixed upstream but not in it.
|
||||||
|
# APT_REFRESH: the upgrade only runs when this layer is rebuilt, and a build cache keeps it for as long
|
||||||
|
# as nothing above it changes — a publish shipped an image whose curl and librsvg were a release behind
|
||||||
|
# fixes already in the archive. The release build passes the day here, so the layer is rebuilt at
|
||||||
|
# least daily; empty (the default, a self-hosted build) leaves the cache alone.
|
||||||
ARG DENO_VERSION=v2.9.1
|
ARG DENO_VERSION=v2.9.1
|
||||||
RUN apt-get update \
|
ARG APT_REFRESH=
|
||||||
|
RUN echo "apt refresh: ${APT_REFRESH:-cached}" \
|
||||||
|
&& apt-get update \
|
||||||
&& apt-get upgrade -y --no-install-recommends \
|
&& apt-get upgrade -y --no-install-recommends \
|
||||||
&& apt-get install -y --no-install-recommends ffmpeg intel-media-va-driver unzip curl ca-certificates \
|
&& apt-get install -y --no-install-recommends ffmpeg intel-media-va-driver unzip curl ca-certificates \
|
||||||
&& curl -fsSL "https://github.com/denoland/deno/releases/download/${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o /tmp/deno.zip \
|
&& curl -fsSL "https://github.com/denoland/deno/releases/download/${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o /tmp/deno.zip \
|
||||||
|
|||||||
@@ -22,9 +22,11 @@ blocker and SponsorBlock keep working.
|
|||||||
tags to slice the feed by.
|
tags to slice the feed by.
|
||||||
- **Playlists with two-way YouTube sync** — build them locally, keep them in sync in both directions.
|
- **Playlists with two-way YouTube sync** — build them locally, keep them in sync in both directions.
|
||||||
- **In-app player** with resume, plus keyboard/scroll controls.
|
- **In-app player** with resume, plus keyboard/scroll controls.
|
||||||
- **Download Center** — save videos to the server with yt-dlp in a Plex-friendly layout (format
|
- **Files** — one library for everything you keep: paste a link anywhere in the app to download it
|
||||||
presets, per-user storage quota), trim / crop / split & join them in a built-in editor, then save
|
with yt-dlp (format presets, per-user storage quota, the running downloads in a tray), upload your
|
||||||
to your device, share with another user, or hand out a public watch link.
|
own files, sort them into folders, trim / crop / split & join videos in a built-in editor, play them
|
||||||
|
where you left off, then save to your device, share with another user, or hand out a public watch
|
||||||
|
link that plays on any phone.
|
||||||
- **Plex library** *(optional)* — point Siftlode at your own Plex server to browse and play its
|
- **Plex library** *(optional)* — point Siftlode at your own Plex server to browse and play its
|
||||||
movies and shows alongside your feed, with watch state kept in sync.
|
movies and shows alongside your feed, with watch state kept in sync.
|
||||||
- **Multi-user** with per-user private state, a shared catalog, and a fair daily API-quota guard.
|
- **Multi-user** with per-user private state, a shared catalog, and a fair daily API-quota guard.
|
||||||
|
|||||||
@@ -437,14 +437,6 @@
|
|||||||
"lineCount": 1
|
"lineCount": 1
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 12,
|
|
||||||
"endColumn": 13,
|
|
||||||
"lineCount": 10
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"code": "reportAttributeAccessIssue",
|
"code": "reportAttributeAccessIssue",
|
||||||
"range": {
|
"range": {
|
||||||
@@ -735,38 +727,6 @@
|
|||||||
}
|
}
|
||||||
],
|
],
|
||||||
"./app/routes/downloads.py": [
|
"./app/routes/downloads.py": [
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 29,
|
|
||||||
"endColumn": 33,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 37,
|
|
||||||
"endColumn": 47,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 29,
|
|
||||||
"endColumn": 33,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 40,
|
|
||||||
"endColumn": 50,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"code": "reportOptionalMemberAccess",
|
"code": "reportOptionalMemberAccess",
|
||||||
"range": {
|
"range": {
|
||||||
@@ -806,22 +766,6 @@
|
|||||||
"endColumn": 54,
|
"endColumn": 54,
|
||||||
"lineCount": 1
|
"lineCount": 1
|
||||||
}
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 29,
|
|
||||||
"endColumn": 33,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 40,
|
|
||||||
"endColumn": 50,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"./app/routes/playlists.py": [
|
"./app/routes/playlists.py": [
|
||||||
@@ -1142,22 +1086,6 @@
|
|||||||
"endColumn": 48,
|
"endColumn": 48,
|
||||||
"lineCount": 1
|
"lineCount": 1
|
||||||
}
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 39,
|
|
||||||
"endColumn": 45,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 38,
|
|
||||||
"endColumn": 43,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"./app/routes/plex/playback.py": [
|
"./app/routes/plex/playback.py": [
|
||||||
@@ -1228,16 +1156,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"./app/routes/public.py": [
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 57,
|
|
||||||
"endColumn": 71,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"./app/routes/scheduler.py": [
|
"./app/routes/scheduler.py": [
|
||||||
{
|
{
|
||||||
"code": "reportArgumentType",
|
"code": "reportArgumentType",
|
||||||
@@ -1463,14 +1381,6 @@
|
|||||||
"lineCount": 1
|
"lineCount": 1
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 38,
|
|
||||||
"endColumn": 43,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"code": "reportArgumentType",
|
"code": "reportArgumentType",
|
||||||
"range": {
|
"range": {
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ Create Date: 2026-07-07
|
|||||||
|
|
||||||
Some sources (a direct media URL like a reddit v.redd.it HLS manifest) carry no thumbnail, so the
|
Some sources (a direct media URL like a reddit v.redd.it HLS manifest) carry no thumbnail, so the
|
||||||
download shows a blank image box. The worker now extracts a representative frame with ffmpeg and
|
download shows a blank image box. The worker now extracts a representative frame with ffmpeg and
|
||||||
saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
|
saves it as a poster sidecar; `poster_path` (relative to files_root) records it so the card /
|
||||||
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
|
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
"""library: a resume point per user and library item
|
||||||
|
|
||||||
|
The library player opened every item at 0:00. `library_states` keeps where a user stopped, per job
|
||||||
|
(their own download, or one shared with them) — the same shape as `video_states`' position, without a
|
||||||
|
status: the library has no "watched". It starts empty; the downgrade drops it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0099_library_states"
|
||||||
|
down_revision: str | None = "0098_drop_demo_account"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"library_states",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"job_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("download_jobs.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("position_seconds", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("progress_updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_library_states_job_id", "library_states", ["job_id"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_library_states_job_id", table_name="library_states")
|
||||||
|
op.drop_table("library_states")
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
"""plex_items: drop `markers`, a column nothing writes or reads
|
||||||
|
|
||||||
|
Intro/credits markers are read live from Plex for the item the player opens (with Plex's `final` flag
|
||||||
|
on the block that ends it); this cached copy was never filled — every row is NULL. The downgrade
|
||||||
|
restores the empty column.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0100_drop_plex_item_markers"
|
||||||
|
down_revision: str | None = "0099_library_states"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.drop_column("plex_items", "markers")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.add_column("plex_items", sa.Column("markers", sa.JSON(), nullable=True))
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""Plex summaries and taglines as plain text
|
||||||
|
|
||||||
|
Revision ID: 0101_plex_text_plain
|
||||||
|
Revises: 0100_drop_plex_item_markers
|
||||||
|
Create Date: 2026-10-08
|
||||||
|
|
||||||
|
Some Plex agents write a summary as HTML (`<p>…</p>`, `á`), and the app showed the tags as
|
||||||
|
text. The sync now stores every summary and tagline through `plain_text` (app/plex/plaintext.py);
|
||||||
|
this cleans the rows it wrote before, through the same function, so nothing waits for a full resync.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
from app.plex.plaintext import plain_text
|
||||||
|
|
||||||
|
revision: str = "0101_plex_text_plain"
|
||||||
|
down_revision: str | None = "0100_drop_plex_item_markers"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
_COLUMNS = {
|
||||||
|
"plex_items": ("summary", "summary_hu", "tagline", "tagline_hu"),
|
||||||
|
"plex_shows": ("summary", "summary_hu", "tagline", "tagline_hu"),
|
||||||
|
"plex_collections": ("summary",),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
conn = op.get_bind()
|
||||||
|
for table, columns in _COLUMNS.items():
|
||||||
|
for col in columns:
|
||||||
|
rows = conn.execute(
|
||||||
|
sa.text(f"SELECT id, {col} FROM {table} WHERE {col} ~ '[<&]'")
|
||||||
|
).fetchall()
|
||||||
|
updates = [
|
||||||
|
{"id": rid, "v": clean}
|
||||||
|
for rid, value in rows
|
||||||
|
if (clean := plain_text(value)) != value
|
||||||
|
]
|
||||||
|
if updates:
|
||||||
|
conn.execute(sa.text(f"UPDATE {table} SET {col} = :v WHERE id = :id"), updates)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
pass # the markup is not wanted back; the next sync would clean it again
|
||||||
@@ -0,0 +1,193 @@
|
|||||||
|
"""The Files library: folders, and a file's retention on the user's job
|
||||||
|
|
||||||
|
Revision ID: 0102_file_folders
|
||||||
|
Revises: 0101_plex_text_plain
|
||||||
|
Create Date: 2026-10-11
|
||||||
|
|
||||||
|
Every download, clip and upload of a user now sits in a folder of their Files library
|
||||||
|
(`app.downloads.folders`): a source folder made per origin (YouTube, Plex, uploads, one per other
|
||||||
|
site) or one of their own. How long a file stays moves from the shared asset to each user's job
|
||||||
|
(`download_jobs.expires_at`, stamped from the folder's `retention_days`), so one user keeping a file
|
||||||
|
no longer depends on another's copy; the asset's own `expires_at` / `gc_notified` go.
|
||||||
|
|
||||||
|
This places every existing job in its source folder and carries each done job's expiry over from
|
||||||
|
the asset it holds (an upload keeps none). The source-key rule is copied here, frozen, from
|
||||||
|
`folders.source_key_for` as it is today — a migration must not follow the code's later changes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0102_file_folders"
|
||||||
|
down_revision: str | None = "0101_plex_text_plain"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
|
||||||
|
_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
|
||||||
|
_DEFAULT_RETENTION_DAYS = 30 # config.download_retention_days on the day this was written
|
||||||
|
|
||||||
|
|
||||||
|
def _site_key(url: str) -> str:
|
||||||
|
host = (urlsplit(url).hostname or "").lower().rstrip(".")
|
||||||
|
for door in _FRONT_DOORS:
|
||||||
|
if host.startswith(door) and host.count(".") > 1:
|
||||||
|
host = host[len(door) :]
|
||||||
|
break
|
||||||
|
if host in _YOUTUBE_HOSTS:
|
||||||
|
return "youtube"
|
||||||
|
return f"site:{host or 'unknown'}"
|
||||||
|
|
||||||
|
|
||||||
|
def _asset_key(kind: str | None, ref: str | None) -> str:
|
||||||
|
if kind == "youtube":
|
||||||
|
return "youtube"
|
||||||
|
if kind == "url" and ref:
|
||||||
|
return _site_key(ref)
|
||||||
|
return "upload"
|
||||||
|
|
||||||
|
|
||||||
|
def _key(job, jobs_by_id, assets) -> str:
|
||||||
|
kind = job.source_kind
|
||||||
|
if kind == "youtube":
|
||||||
|
return "youtube"
|
||||||
|
if kind == "upload":
|
||||||
|
return "upload"
|
||||||
|
if kind == "clip":
|
||||||
|
spec = job.edit_spec or {}
|
||||||
|
return "plex" if spec.get("source") == "plex" else "youtube"
|
||||||
|
if kind == "url":
|
||||||
|
return _site_key(job.source_ref)
|
||||||
|
if kind == "edit":
|
||||||
|
parent = jobs_by_id.get(job.source_job_id)
|
||||||
|
if parent is not None and parent.id != job.id:
|
||||||
|
return _key(parent, jobs_by_id, assets)
|
||||||
|
asset = assets.get(job.source_asset_id)
|
||||||
|
if asset is not None:
|
||||||
|
return _asset_key(asset.source_kind, asset.source_ref)
|
||||||
|
return "upload"
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"file_folders",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("kind", sa.String(16), nullable=False),
|
||||||
|
sa.Column("source_key", sa.String(260)), # "site:" + a host (<= 253)
|
||||||
|
sa.Column("name", sa.String(255)),
|
||||||
|
sa.Column("retention_days", sa.Integer()),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||||
|
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)",
|
||||||
|
name="ck_file_folder_kind",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index("ix_file_folders_user_id", "file_folders", ["user_id"])
|
||||||
|
op.create_index(
|
||||||
|
"uq_file_folder_own_name",
|
||||||
|
"file_folders",
|
||||||
|
["user_id", sa.text("lower(name)")],
|
||||||
|
unique=True,
|
||||||
|
postgresql_where=sa.text("name IS NOT NULL"),
|
||||||
|
)
|
||||||
|
|
||||||
|
op.add_column(
|
||||||
|
"download_jobs",
|
||||||
|
sa.Column(
|
||||||
|
"folder_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("file_folders.id", ondelete="RESTRICT"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"download_jobs",
|
||||||
|
sa.Column("kept", sa.Boolean(), server_default="false", nullable=False),
|
||||||
|
)
|
||||||
|
op.add_column("download_jobs", sa.Column("expires_at", sa.DateTime(timezone=True)))
|
||||||
|
op.add_column(
|
||||||
|
"download_jobs",
|
||||||
|
sa.Column("expiry_warned", sa.Boolean(), server_default="false", nullable=False),
|
||||||
|
)
|
||||||
|
op.create_index("ix_download_jobs_folder_id", "download_jobs", ["folder_id"])
|
||||||
|
op.create_index("ix_download_jobs_expires_at", "download_jobs", ["expires_at"])
|
||||||
|
|
||||||
|
conn = op.get_bind()
|
||||||
|
raw = conn.execute(
|
||||||
|
sa.text("SELECT value FROM system_config WHERE key = 'download_retention_days'")
|
||||||
|
).scalar()
|
||||||
|
retention = int(raw) if raw not in (None, "") else _DEFAULT_RETENTION_DAYS
|
||||||
|
|
||||||
|
jobs = conn.execute(
|
||||||
|
sa.text(
|
||||||
|
"SELECT j.id, j.user_id, j.source_kind, j.source_ref, j.edit_spec, j.source_job_id,"
|
||||||
|
" j.source_asset_id, j.status, j.asset_id, a.expires_at AS asset_expires_at"
|
||||||
|
" FROM download_jobs j LEFT JOIN media_assets a ON a.id = j.asset_id"
|
||||||
|
)
|
||||||
|
).fetchall()
|
||||||
|
jobs_by_id = {j.id: j for j in jobs}
|
||||||
|
assets = {
|
||||||
|
a.id: a
|
||||||
|
for a in conn.execute(sa.text("SELECT id, source_kind, source_ref FROM media_assets"))
|
||||||
|
}
|
||||||
|
|
||||||
|
folder_ids: dict[tuple[int, str], int] = {}
|
||||||
|
for job in jobs:
|
||||||
|
key = _key(job, jobs_by_id, assets)
|
||||||
|
slot = (job.user_id, key)
|
||||||
|
if slot not in folder_ids:
|
||||||
|
folder_ids[slot] = conn.execute(
|
||||||
|
sa.text(
|
||||||
|
"INSERT INTO file_folders (user_id, kind, source_key, retention_days)"
|
||||||
|
" VALUES (:u, 'source', :k, :r) RETURNING id"
|
||||||
|
),
|
||||||
|
{"u": job.user_id, "k": key, "r": None if key == "upload" else retention},
|
||||||
|
).scalar_one()
|
||||||
|
# A done file keeps the expiry it had (its asset's); an upload had none and keeps none.
|
||||||
|
expires = job.asset_expires_at if job.status == "done" and key != "upload" else None
|
||||||
|
conn.execute(
|
||||||
|
sa.text("UPDATE download_jobs SET folder_id = :f, expires_at = :e WHERE id = :id"),
|
||||||
|
{"f": folder_ids[slot], "e": expires, "id": job.id},
|
||||||
|
)
|
||||||
|
|
||||||
|
op.drop_column("media_assets", "expires_at")
|
||||||
|
op.drop_column("media_assets", "gc_notified")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.add_column("media_assets", sa.Column("expires_at", sa.DateTime(timezone=True)))
|
||||||
|
op.add_column(
|
||||||
|
"media_assets",
|
||||||
|
sa.Column("gc_notified", sa.Boolean(), server_default="false", nullable=False),
|
||||||
|
)
|
||||||
|
# The asset goes when its last holder would: the latest of its holders' expiries (a holder that
|
||||||
|
# never expires keeps the asset unexpired).
|
||||||
|
op.execute(
|
||||||
|
"UPDATE media_assets a SET expires_at = sub.e FROM ("
|
||||||
|
" SELECT asset_id, CASE WHEN bool_or(expires_at IS NULL) THEN NULL ELSE max(expires_at) END"
|
||||||
|
" AS e FROM download_jobs WHERE asset_id IS NOT NULL AND status = 'done' GROUP BY asset_id"
|
||||||
|
") sub WHERE a.id = sub.asset_id AND a.source_kind <> 'upload'"
|
||||||
|
)
|
||||||
|
op.drop_index("ix_download_jobs_expires_at", table_name="download_jobs")
|
||||||
|
op.drop_index("ix_download_jobs_folder_id", table_name="download_jobs")
|
||||||
|
op.drop_column("download_jobs", "expiry_warned")
|
||||||
|
op.drop_column("download_jobs", "expires_at")
|
||||||
|
op.drop_column("download_jobs", "kept")
|
||||||
|
op.drop_column("download_jobs", "folder_id")
|
||||||
|
op.drop_index("uq_file_folder_own_name", table_name="file_folders")
|
||||||
|
op.drop_index("ix_file_folders_user_id", table_name="file_folders")
|
||||||
|
op.drop_table("file_folders")
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
"""The Files library: a file's SHA-256, and the expired rows retired
|
||||||
|
|
||||||
|
Revision ID: 0103_files_sha256
|
||||||
|
Revises: 0102_file_folders
|
||||||
|
Create Date: 2026-10-11
|
||||||
|
|
||||||
|
`media_assets.sha256` holds the file's hash once it is known (an upload is hashed while it streams
|
||||||
|
in, any other file on the first ask), so a machine fetching a file can check what it got.
|
||||||
|
|
||||||
|
An expired file now leaves its user's library (`downloads.gc`). Before, an expired download stayed
|
||||||
|
as a fileless "expired · download again" row (a done job with no asset). Those rows hold nothing on
|
||||||
|
disk and the library no longer shows them, so they are deleted here rather than left behind
|
||||||
|
unseen; their shares and public links go with them (ON DELETE CASCADE), and those already served
|
||||||
|
nothing. The downgrade cannot bring them back.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0103_files_sha256"
|
||||||
|
down_revision: str | None = "0102_file_folders"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("media_assets", sa.Column("sha256", sa.String(64)))
|
||||||
|
op.execute("DELETE FROM download_jobs WHERE status = 'done' AND asset_id IS NULL")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("media_assets", "sha256")
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
"""A file's "Added" is when it became done
|
||||||
|
|
||||||
|
Revision ID: 0104_job_ready_at
|
||||||
|
Revises: 0103_files_sha256
|
||||||
|
Create Date: 2026-10-12
|
||||||
|
|
||||||
|
`download_jobs.ready_at` is when a job became done, and the Files library sorts and dates by it: a
|
||||||
|
download that failed for days and then went through used to sit days down the list, under its
|
||||||
|
request date. Every path to done stamps it, held by `ck_download_jobs_ready_at`.
|
||||||
|
|
||||||
|
A done job from before has no record of that moment (its "ready to watch" notice never named the
|
||||||
|
job, so it cannot be matched back), so it takes its request date, which is where it already sat.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0104_job_ready_at"
|
||||||
|
down_revision: str | None = "0103_files_sha256"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("download_jobs", sa.Column("ready_at", sa.DateTime(timezone=True)))
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE download_jobs SET ready_at = created_at WHERE status = 'done'
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.create_check_constraint(
|
||||||
|
"ck_download_jobs_ready_at", "download_jobs", "status <> 'done' OR ready_at IS NOT NULL"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_constraint("ck_download_jobs_ready_at", "download_jobs", type_="check")
|
||||||
|
op.drop_column("download_jobs", "ready_at")
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
"""A file remembers where it came from
|
||||||
|
|
||||||
|
Revision ID: 0105_job_source_key
|
||||||
|
Revises: 0104_job_ready_at
|
||||||
|
Create Date: 2026-10-12
|
||||||
|
|
||||||
|
`download_jobs.source_key` is the origin a file came from — the source folder it may go back to —
|
||||||
|
recorded when it is placed. An edit used to work its origin out through its parent each time, and
|
||||||
|
once the parent was deleted or expired it fell back to "upload": moved into an own folder, it could
|
||||||
|
no longer go back to its folder, and deleting that folder sent it to Uploads to stay forever.
|
||||||
|
|
||||||
|
A file in a source folder takes that folder's key. One already in an own folder is left NULL and
|
||||||
|
keeps being worked out (`folders.source_key_for`), as before.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0105_job_source_key"
|
||||||
|
down_revision: str | None = "0104_job_ready_at"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("download_jobs", sa.Column("source_key", sa.String(260)))
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE download_jobs j SET source_key = f.source_key
|
||||||
|
FROM file_folders f
|
||||||
|
WHERE f.id = j.folder_id AND f.kind = 'source'
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("download_jobs", "source_key")
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
"""Machine access to the Files library: API tokens bound to folders, and the Hub folder
|
||||||
|
|
||||||
|
Revision ID: 0106_api_tokens
|
||||||
|
Revises: 0105_job_source_key
|
||||||
|
Create Date: 2026-10-14
|
||||||
|
|
||||||
|
`api_tokens` is a user's token for a machine (a script, the hub): only its SHA-256 is kept, plus the
|
||||||
|
first characters to recognise it by, and whether its user was told it is about to run out. `api_token_grants` says what it may do in which folder — list,
|
||||||
|
read, write, delete — so a token reaches nothing it was not given, folder by folder.
|
||||||
|
|
||||||
|
A third folder kind, `hub`: the one folder a user exchanges files with their hub machine through.
|
||||||
|
Like a source folder its name is the client's to render; like an own folder a file may be moved in.
|
||||||
|
One per user.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0106_api_tokens"
|
||||||
|
down_revision: str | None = "0105_job_source_key"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
_KIND_OLD = (
|
||||||
|
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||||
|
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
||||||
|
)
|
||||||
|
_KIND_NEW = _KIND_OLD + " OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)"
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.drop_constraint("ck_file_folder_kind", "file_folders", type_="check")
|
||||||
|
op.create_check_constraint("ck_file_folder_kind", "file_folders", _KIND_NEW)
|
||||||
|
op.create_index(
|
||||||
|
"uq_file_folder_hub",
|
||||||
|
"file_folders",
|
||||||
|
["user_id"],
|
||||||
|
unique=True,
|
||||||
|
postgresql_where=sa.text("kind = 'hub'"),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"api_tokens",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
),
|
||||||
|
sa.Column("name", sa.String(80), nullable=False),
|
||||||
|
sa.Column("prefix", sa.String(16), nullable=False),
|
||||||
|
sa.Column("token_hash", sa.String(64), nullable=False, unique=True),
|
||||||
|
sa.Column("expires_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.Column("last_used_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.Column("expiry_warned", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||||
|
sa.Column(
|
||||||
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"api_token_grants",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"token_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("api_tokens.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"folder_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("file_folders.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
),
|
||||||
|
sa.Column("can_list", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||||
|
sa.Column("can_read", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||||
|
sa.Column("can_write", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||||
|
sa.Column("can_delete", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||||
|
sa.UniqueConstraint("token_id", "folder_id", name="uq_api_token_grant"),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"can_list OR can_read OR can_write OR can_delete", name="ck_api_token_grant_some"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("api_token_grants")
|
||||||
|
op.drop_table("api_tokens")
|
||||||
|
op.drop_index("uq_file_folder_hub", table_name="file_folders")
|
||||||
|
# A Hub folder may hold files (their foreign key RESTRICTs a delete): it becomes an own folder,
|
||||||
|
# named "Hub <id>" — or "Hub <id> (2)", … when its user already has a folder of that name,
|
||||||
|
# which `uq_file_folder_own_name` (per user, case-blind) would refuse.
|
||||||
|
conn = op.get_bind()
|
||||||
|
hubs = conn.execute(sa.text("SELECT id, user_id FROM file_folders WHERE kind = 'hub'")).all()
|
||||||
|
for folder_id, user_id in hubs:
|
||||||
|
name, n = f"Hub {folder_id}", 1
|
||||||
|
while conn.execute(
|
||||||
|
sa.text("SELECT 1 FROM file_folders WHERE user_id = :u AND lower(name) = lower(:n)"),
|
||||||
|
{"u": user_id, "n": name},
|
||||||
|
).first():
|
||||||
|
n += 1
|
||||||
|
name = f"Hub {folder_id} ({n})"
|
||||||
|
conn.execute(
|
||||||
|
sa.text("UPDATE file_folders SET kind = 'own', name = :n WHERE id = :i"),
|
||||||
|
{"n": name, "i": folder_id},
|
||||||
|
)
|
||||||
|
op.drop_constraint("ck_file_folder_kind", "file_folders", type_="check")
|
||||||
|
op.create_check_constraint("ck_file_folder_kind", "file_folders", _KIND_OLD)
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
"""Share-link tokens no longer stored in the clear
|
||||||
|
|
||||||
|
Revision ID: 0107_link_token_at_rest
|
||||||
|
Revises: 0106_api_tokens
|
||||||
|
Create Date: 2026-10-15
|
||||||
|
|
||||||
|
A `/watch` link's token is the credential itself, and it sat in `download_links.token` as written,
|
||||||
|
so a copy of the database was every shared file. Now a link is found by the token's SHA-256
|
||||||
|
(`token_hash`), and the token is kept only Fernet-encrypted (`token_enc`, TOKEN_ENCRYPTION_KEY) so its
|
||||||
|
owner can copy the same link again. Without a key the copy is not kept: the link still opens, its
|
||||||
|
address just cannot be shown again (`app/downloads/links.py`).
|
||||||
|
|
||||||
|
Every existing link is converted here, so the addresses already sent keep working. The downgrade
|
||||||
|
writes the tokens back from their encrypted copies; a link with none left (no key, or a key since
|
||||||
|
changed) gets a fresh random token, which is the same as revoking it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import secrets
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
from app import security
|
||||||
|
|
||||||
|
revision: str = "0107_link_token_at_rest"
|
||||||
|
down_revision: str | None = "0106_api_tokens"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("download_links", sa.Column("token_hash", sa.String(64)))
|
||||||
|
op.add_column("download_links", sa.Column("token_enc", sa.Text()))
|
||||||
|
conn = op.get_bind()
|
||||||
|
sealing = security.can_encrypt()
|
||||||
|
for link_id, token in conn.execute(sa.text("SELECT id, token FROM download_links")).fetchall():
|
||||||
|
conn.execute(
|
||||||
|
sa.text("UPDATE download_links SET token_hash = :h, token_enc = :e WHERE id = :id"),
|
||||||
|
{
|
||||||
|
"id": link_id,
|
||||||
|
"h": hashlib.sha256(token.encode()).hexdigest(),
|
||||||
|
"e": security.encrypt(token) if sealing else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
op.alter_column("download_links", "token_hash", nullable=False)
|
||||||
|
op.create_unique_constraint("uq_download_link_token_hash", "download_links", ["token_hash"])
|
||||||
|
op.drop_constraint("uq_download_link_token", "download_links", type_="unique")
|
||||||
|
op.drop_column("download_links", "token")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.add_column("download_links", sa.Column("token", sa.String(64)))
|
||||||
|
conn = op.get_bind()
|
||||||
|
rows = conn.execute(sa.text("SELECT id, token_enc FROM download_links")).fetchall()
|
||||||
|
for link_id, enc in rows:
|
||||||
|
token = security.decrypt_optional(enc) or secrets.token_urlsafe(24)
|
||||||
|
conn.execute(
|
||||||
|
sa.text("UPDATE download_links SET token = :t WHERE id = :id"),
|
||||||
|
{"id": link_id, "t": token},
|
||||||
|
)
|
||||||
|
op.alter_column("download_links", "token", nullable=False)
|
||||||
|
op.create_unique_constraint("uq_download_link_token", "download_links", ["token"])
|
||||||
|
op.drop_constraint("uq_download_link_token_hash", "download_links", type_="unique")
|
||||||
|
op.drop_column("download_links", "token_enc")
|
||||||
|
op.drop_column("download_links", "token_hash")
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""A share link may open a folder
|
||||||
|
|
||||||
|
Revision ID: 0108_folder_links
|
||||||
|
Revises: 0107_link_token_at_rest
|
||||||
|
Create Date: 2026-10-15
|
||||||
|
|
||||||
|
A `/watch` link opened one file. Now it may open one of its owner's own folders instead: the link's
|
||||||
|
page lists what is in the folder when it is opened. `folder_id` sits beside `job_id`, exactly one of
|
||||||
|
the two set; a folder's links go with the folder.
|
||||||
|
|
||||||
|
The downgrade has nowhere to keep a folder link, so it deletes them: they stop opening.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0108_folder_links"
|
||||||
|
down_revision: str | None = "0107_link_token_at_rest"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"download_links",
|
||||||
|
sa.Column("folder_id", sa.Integer(), sa.ForeignKey("file_folders.id", ondelete="CASCADE")),
|
||||||
|
)
|
||||||
|
op.create_index("ix_download_links_folder_id", "download_links", ["folder_id"])
|
||||||
|
op.alter_column("download_links", "job_id", nullable=True)
|
||||||
|
op.create_check_constraint(
|
||||||
|
"ck_download_link_target", "download_links", "(job_id IS NULL) <> (folder_id IS NULL)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("DELETE FROM download_links WHERE folder_id IS NOT NULL")
|
||||||
|
op.drop_constraint("ck_download_link_target", "download_links", type_="check")
|
||||||
|
op.alter_column("download_links", "job_id", nullable=False)
|
||||||
|
op.drop_index("ix_download_links_folder_id", table_name="download_links")
|
||||||
|
op.drop_column("download_links", "folder_id")
|
||||||
@@ -20,7 +20,8 @@ Three layers, by *who* runs the code — not by feature:
|
|||||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline;
|
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline;
|
||||||
`issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker.
|
`issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker.
|
||||||
|
|
||||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth` (with `apitokens`, a machine's
|
||||||
|
folder-scoped way into the Files library), `sysconfig`, `quota`, `audit`,
|
||||||
`notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging
|
`notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging
|
||||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||||
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
||||||
|
|||||||
@@ -0,0 +1,273 @@
|
|||||||
|
"""Machine access to the Files library: API tokens, and what each one may do where.
|
||||||
|
|
||||||
|
A user makes a token for a machine of theirs — their hub, a script — and it reaches their library as
|
||||||
|
`Authorization: Bearer sfl_…`. Only on `/api/files`: the routes there take `files_caller` instead of
|
||||||
|
`current_user`, and nothing else does (a test holds the route table to that), so a token can never
|
||||||
|
manage the account, the tokens, or anything outside the library. And only in the folders its grants
|
||||||
|
name, with the rights they give: list, read, write, delete. Every route asks the one question,
|
||||||
|
`FilesCaller.may(folder_id, right)`; a signed-in user may everything in their own folders.
|
||||||
|
|
||||||
|
The token is `sfl_` + 32 random bytes in base62 + a CRC32 of them: a mistyped or cut-off token is
|
||||||
|
refused without a database read, and a secret scanner can tell one by its shape. Only its SHA-256 is
|
||||||
|
kept (`security.hash_token`) — the user sees it once, when it is made — with its first characters to
|
||||||
|
tell tokens apart by. It is looked up by that hash on every request: no signed claims, so a deleted
|
||||||
|
token stops working at once. A token about to run out warns its user once, a week before
|
||||||
|
(`warn_expiring`), so a machine does not lose its key unannounced.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import secrets
|
||||||
|
import zlib
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
from fastapi import Depends, Request
|
||||||
|
from sqlalchemy import delete, or_, select, update
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import audit, debugsession, logs
|
||||||
|
from app.audit import AuditAction
|
||||||
|
from app.auth import current_user
|
||||||
|
from app.db import get_db
|
||||||
|
from app.models import ApiToken, ApiTokenGrant, User
|
||||||
|
from app.notifications import create_notification
|
||||||
|
from app.ratelimit import RateLimiter, client_key
|
||||||
|
from app.refusal import refused
|
||||||
|
from app.security import hash_token
|
||||||
|
from app.utils import iso
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
PREFIX = "sfl_"
|
||||||
|
_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||||
|
_BODY_BYTES = 32
|
||||||
|
_BODY_LEN = 43 # 62**43 > 2**256: every 32-byte value fits
|
||||||
|
_CHECK_LEN = 6 # 62**6 > 2**32: every CRC32 fits
|
||||||
|
TOKEN_LEN = len(PREFIX) + _BODY_LEN + _CHECK_LEN
|
||||||
|
# What a token is shown by once it is made: enough to tell a user's tokens apart, far too little to
|
||||||
|
# narrow a guess (8 of its 43 random characters).
|
||||||
|
SHOWN_LEN = len(PREFIX) + 8
|
||||||
|
|
||||||
|
# A machine that runs away (a loop in a script) is held back per token; a caller trying tokens is
|
||||||
|
# held back per client, counting only the refusals.
|
||||||
|
_calls = RateLimiter(600, 60)
|
||||||
|
_bad = RateLimiter(20, 300)
|
||||||
|
# How stale `last_used_at` may get before a request writes it again: a busy script costs one write a
|
||||||
|
# minute, not one per request.
|
||||||
|
_TOUCH_EVERY = timedelta(minutes=1)
|
||||||
|
# How long before a token runs out its user is told (`warn_expiring`); the token list marks it
|
||||||
|
# "expires soon" for the same span.
|
||||||
|
EXPIRY_WARNING = timedelta(days=7)
|
||||||
|
|
||||||
|
|
||||||
|
class Right(StrEnum):
|
||||||
|
LIST = "list"
|
||||||
|
READ = "read"
|
||||||
|
WRITE = "write"
|
||||||
|
DELETE = "delete"
|
||||||
|
|
||||||
|
|
||||||
|
# Each right's column on a grant. A dict over the enum, so a new right without a column fails here.
|
||||||
|
GRANT_COLUMNS = {
|
||||||
|
Right.LIST: "can_list",
|
||||||
|
Right.READ: "can_read",
|
||||||
|
Right.WRITE: "can_write",
|
||||||
|
Right.DELETE: "can_delete",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _b62(value: int, width: int) -> str:
|
||||||
|
out = []
|
||||||
|
for _ in range(width):
|
||||||
|
value, digit = divmod(value, 62)
|
||||||
|
out.append(_ALPHABET[digit])
|
||||||
|
return "".join(reversed(out))
|
||||||
|
|
||||||
|
|
||||||
|
def _check(body: str) -> str:
|
||||||
|
return _b62(zlib.crc32(body.encode()), _CHECK_LEN)
|
||||||
|
|
||||||
|
|
||||||
|
def mint() -> str:
|
||||||
|
"""A new token, in full. The caller stores its hash and shows it to the user, once."""
|
||||||
|
body = _b62(int.from_bytes(secrets.token_bytes(_BODY_BYTES)), _BODY_LEN)
|
||||||
|
return PREFIX + body + _check(body)
|
||||||
|
|
||||||
|
|
||||||
|
def well_formed(raw: str) -> bool:
|
||||||
|
"""Whether `raw` has a token's shape and its checksum adds up."""
|
||||||
|
if len(raw) != TOKEN_LEN or not raw.startswith(PREFIX):
|
||||||
|
return False
|
||||||
|
body, check = raw[len(PREFIX) : -_CHECK_LEN], raw[-_CHECK_LEN:]
|
||||||
|
# The check too: compare_digest refuses (TypeError, a 500) a str with anything non-ASCII in it.
|
||||||
|
if any(c not in _ALPHABET for c in body + check):
|
||||||
|
return False
|
||||||
|
return secrets.compare_digest(_check(body), check)
|
||||||
|
|
||||||
|
|
||||||
|
def grant_rights(grant: ApiTokenGrant) -> frozenset[Right]:
|
||||||
|
return frozenset(r for r, col in GRANT_COLUMNS.items() if getattr(grant, col))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FilesCaller:
|
||||||
|
"""Who is asking the Files library, and where they may do what. `grants` is None for a signed-in
|
||||||
|
user (everything in their own folders); for a token, folder id → its rights there."""
|
||||||
|
|
||||||
|
user: User
|
||||||
|
token_id: int | None = None
|
||||||
|
token_prefix: str | None = None
|
||||||
|
grants: Mapping[int, frozenset[Right]] | None = None
|
||||||
|
|
||||||
|
def may(self, folder_id: int | None, right: Right) -> bool:
|
||||||
|
if self.grants is None:
|
||||||
|
return True
|
||||||
|
return folder_id is not None and right in self.grants.get(folder_id, frozenset())
|
||||||
|
|
||||||
|
def folders_with(self, right: Right) -> set[int] | None:
|
||||||
|
"""The folders `right` is granted in; None = any of the user's own."""
|
||||||
|
if self.grants is None:
|
||||||
|
return None
|
||||||
|
return {f for f, rights in self.grants.items() if right in rights}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def via(self) -> str | None:
|
||||||
|
"""How an audit row says a token did it, None when the user did."""
|
||||||
|
return None if self.token_prefix is None else f"api token {self.token_prefix}…"
|
||||||
|
|
||||||
|
|
||||||
|
def bearer_token(request: Request) -> str | None:
|
||||||
|
"""The token of an `Authorization: Bearer` header; None when the request carries none (another
|
||||||
|
scheme counts as none — it is not ours to judge)."""
|
||||||
|
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||||
|
if scheme.lower() != "bearer":
|
||||||
|
return None
|
||||||
|
return value.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def session_caller(request: Request, db: Session = Depends(get_db)) -> User | None:
|
||||||
|
"""The signed-in user, or None when the request brings a token instead: a token wins over a
|
||||||
|
cookie, so a machine's request is judged by its grants alone, whatever else it carries."""
|
||||||
|
if bearer_token(request) is not None:
|
||||||
|
return None
|
||||||
|
return current_user(request, db)
|
||||||
|
|
||||||
|
|
||||||
|
def files_caller(
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
user: User | None = Depends(session_caller),
|
||||||
|
) -> FilesCaller:
|
||||||
|
"""The `/api/files` caller: a signed-in user, or a valid token's user with its grants."""
|
||||||
|
if user is not None:
|
||||||
|
return FilesCaller(user=user)
|
||||||
|
return _token_caller(request, db, bearer_token(request) or "")
|
||||||
|
|
||||||
|
|
||||||
|
def _refuse_token(key: str, reason: str) -> Exception:
|
||||||
|
_bad.allow(key)
|
||||||
|
log.info(
|
||||||
|
"a request brought an API token that does not hold",
|
||||||
|
extra=logs.event("apitoken.refused", reason=reason),
|
||||||
|
)
|
||||||
|
return refused(401, "not_authenticated")
|
||||||
|
|
||||||
|
|
||||||
|
def _token_caller(request: Request, db: Session, raw: str) -> FilesCaller:
|
||||||
|
key = client_key(request)
|
||||||
|
if _bad.exhausted(key):
|
||||||
|
raise refused(429, "api_token_rate_limited")
|
||||||
|
if not well_formed(raw):
|
||||||
|
raise _refuse_token(key, "malformed")
|
||||||
|
token = db.execute(
|
||||||
|
select(ApiToken).where(ApiToken.token_hash == hash_token(raw))
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if token is None:
|
||||||
|
raise _refuse_token(key, "unknown")
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
if token.expires_at is not None and token.expires_at <= now:
|
||||||
|
raise _refuse_token(key, "expired")
|
||||||
|
user = db.get(User, token.user_id)
|
||||||
|
if user is None or not user.is_active or user.is_suspended:
|
||||||
|
raise _refuse_token(key, "account")
|
||||||
|
if not _calls.allow(str(token.id)):
|
||||||
|
raise refused(429, "api_token_rate_limited")
|
||||||
|
grants = {
|
||||||
|
g.folder_id: grant_rights(g)
|
||||||
|
for g in db.execute(select(ApiTokenGrant).where(ApiTokenGrant.token_id == token.id))
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
}
|
||||||
|
caller = FilesCaller(user=user, token_id=token.id, token_prefix=token.prefix, grants=grants)
|
||||||
|
db.execute(
|
||||||
|
update(ApiToken)
|
||||||
|
.where(
|
||||||
|
ApiToken.id == token.id,
|
||||||
|
or_(ApiToken.last_used_at.is_(None), ApiToken.last_used_at < now - _TOUCH_EVERY),
|
||||||
|
)
|
||||||
|
.values(last_used_at=now)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
)
|
||||||
|
# Committed here, ahead of the route: a read-only route never commits, and a write route must
|
||||||
|
# not carry this along into a rollback of its own.
|
||||||
|
db.commit()
|
||||||
|
logs.set_user(user.id, *debugsession.level_for(user, None))
|
||||||
|
request.state.user_id = user.id
|
||||||
|
return caller
|
||||||
|
|
||||||
|
|
||||||
|
def revoke_all(db: Session, user_id: int) -> int:
|
||||||
|
"""Revoke every token of `user_id`, each with its audit row (NOT committed). A token is a
|
||||||
|
credential like a session, so whatever ends every session of an account — a password reset or
|
||||||
|
change, "log out everywhere", a revoked untrusted password — ends its tokens too: one minted
|
||||||
|
from a stolen session must not outlive the owner's answer to that theft. Returns how many."""
|
||||||
|
gone = (
|
||||||
|
db.execute(delete(ApiToken).where(ApiToken.user_id == user_id).returning(ApiToken.id))
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for token_id in gone:
|
||||||
|
audit.record(
|
||||||
|
db,
|
||||||
|
user_id,
|
||||||
|
AuditAction.API_TOKEN_REVOKE,
|
||||||
|
target_type="api_token",
|
||||||
|
target_id=token_id,
|
||||||
|
summary="revoked with every session of the account",
|
||||||
|
)
|
||||||
|
return len(gone)
|
||||||
|
|
||||||
|
|
||||||
|
def expiring(token: ApiToken, now: datetime) -> bool:
|
||||||
|
"""Whether `token` runs out within the warning span (and has not yet)."""
|
||||||
|
return token.expires_at is not None and now < token.expires_at <= now + EXPIRY_WARNING
|
||||||
|
|
||||||
|
|
||||||
|
def warn_expiring(db: Session) -> int:
|
||||||
|
"""Tell each user whose token runs out within `EXPIRY_WARNING` — once per token: the mark and the
|
||||||
|
notice commit together, and a token made again (a new row) warns afresh. Returns how many."""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
hits = db.execute(
|
||||||
|
update(ApiToken)
|
||||||
|
.where(
|
||||||
|
ApiToken.expiry_warned.is_(False),
|
||||||
|
ApiToken.expires_at.is_not(None),
|
||||||
|
ApiToken.expires_at > now,
|
||||||
|
ApiToken.expires_at <= now + EXPIRY_WARNING,
|
||||||
|
)
|
||||||
|
.values(expiry_warned=True)
|
||||||
|
.returning(ApiToken.id, ApiToken.user_id, ApiToken.name, ApiToken.expires_at)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
).all()
|
||||||
|
for token_id, user_id, name, expires_at in hits:
|
||||||
|
create_notification(
|
||||||
|
db,
|
||||||
|
user_id,
|
||||||
|
"api_token_expiring",
|
||||||
|
title=f"API token {name} expires soon", # English fallback; the client names it from data
|
||||||
|
data={"token_id": token_id, "name": name, "expires_at": iso(expires_at)},
|
||||||
|
commit=False,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return len(hits)
|
||||||
@@ -58,6 +58,27 @@ class AuditAction:
|
|||||||
# A user sent a debug session's issue to the tracker — the one act that takes their log
|
# A user sent a debug session's issue to the tracker — the one act that takes their log
|
||||||
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
|
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
|
||||||
ISSUE_CREATE = "issue_create"
|
ISSUE_CREATE = "issue_create"
|
||||||
|
# The Files library (`routes.files`): what a user put in, took out, kept or reorganised. Ids
|
||||||
|
# only — a file's or folder's name is the user's own text, not the admin's to read here.
|
||||||
|
FILE_UPLOAD = "file_upload"
|
||||||
|
FILE_DELETE = "file_delete"
|
||||||
|
FILE_KEEP = "file_keep"
|
||||||
|
FILE_UNKEEP = "file_unkeep"
|
||||||
|
FILE_MOVE = "file_move"
|
||||||
|
FILE_FOLDER_CREATE = "file_folder_create"
|
||||||
|
FILE_FOLDER_RENAME = "file_folder_rename"
|
||||||
|
FILE_FOLDER_RETENTION = "file_folder_retention"
|
||||||
|
FILE_FOLDER_DELETE = "file_folder_delete"
|
||||||
|
# A machine's token to the library (`routes.tokens`): made (its folders and rights) or revoked.
|
||||||
|
# What a token then did is the file rows above, their summary naming it.
|
||||||
|
API_TOKEN_CREATE = "api_token_create"
|
||||||
|
API_TOKEN_REVOKE = "api_token_revoke"
|
||||||
|
# A public `/watch` link to a file or a folder (`routes.downloads`): made, its settings changed,
|
||||||
|
# revoked.
|
||||||
|
# Never its token or password — only whether it has one.
|
||||||
|
DOWNLOAD_LINK_CREATE = "download_link_create"
|
||||||
|
DOWNLOAD_LINK_UPDATE = "download_link_update"
|
||||||
|
DOWNLOAD_LINK_REVOKE = "download_link_revoke"
|
||||||
|
|
||||||
|
|
||||||
def record(
|
def record(
|
||||||
|
|||||||
@@ -92,8 +92,8 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
|||||||
# So this 413 is a refusal a real user meets on a default install, not a theoretical edge, and
|
# So this 413 is a refusal a real user meets on a default install, not a theoretical edge, and
|
||||||
# it has to be as readable as the route's own: the client localizes `too_large` (`errors.tooLarge`)
|
# it has to be as readable as the route's own: the client localizes `too_large` (`errors.tooLarge`)
|
||||||
# exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before
|
# exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before
|
||||||
# sending it at all, off the `upload_max_bytes` that `GET /api/downloads/usage` reports.
|
# sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports.
|
||||||
("POST", "/api/downloads/upload"): settings.download_upload_max_bytes,
|
("POST", "/api/files/upload"): settings.download_upload_max_bytes,
|
||||||
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
|
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+18
-11
@@ -1,7 +1,7 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Literal, get_args, get_origin
|
from typing import Literal, get_args, get_origin
|
||||||
|
|
||||||
from pydantic import Field, ValidationInfo, field_validator, model_validator
|
from pydantic import AliasChoices, Field, ValidationInfo, field_validator, model_validator
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
||||||
# Shipped placeholder; production must override it. Used by the startup guard below.
|
# Shipped placeholder; production must override it. Used by the startup guard below.
|
||||||
@@ -171,9 +171,13 @@ class Settings(BaseSettings):
|
|||||||
feed_default_hidden_live: str = "live,upcoming"
|
feed_default_hidden_live: str = "live,upcoming"
|
||||||
|
|
||||||
# --- Download center (yt-dlp) ---
|
# --- Download center (yt-dlp) ---
|
||||||
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
|
# Root directory (inside the container) for the files library — a Docker volume/bind mount.
|
||||||
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
|
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
|
||||||
download_root: str = "/downloads"
|
# DOWNLOAD_ROOT is its old name: an install still on the old compose file (mounted at
|
||||||
|
# /downloads, DOWNLOAD_ROOT=/downloads) keeps finding its files after it pulls a new image.
|
||||||
|
files_root: str = Field(
|
||||||
|
default="/files", validation_alias=AliasChoices("FILES_ROOT", "DOWNLOAD_ROOT", "files_root")
|
||||||
|
)
|
||||||
# Whether THIS process runs the download worker loop. On for the dedicated worker
|
# Whether THIS process runs the download worker loop. On for the dedicated worker
|
||||||
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
|
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
|
||||||
worker_enabled: bool = False
|
worker_enabled: bool = False
|
||||||
@@ -181,15 +185,18 @@ class Settings(BaseSettings):
|
|||||||
# for YouTube Proof-of-Origin tokens (bypasses bot-detection + unlocks high-quality
|
# for YouTube Proof-of-Origin tokens (bypasses bot-detection + unlocks high-quality
|
||||||
# formats). Empty disables it (yt-dlp still works, but may hit "confirm you're not a bot").
|
# formats). Empty disables it (yt-dlp still works, but may hit "confirm you're not a bot").
|
||||||
download_pot_base_url: str = "http://bgutil-pot:4416"
|
download_pot_base_url: str = "http://bgutil-pot:4416"
|
||||||
# yt-dlp player clients (comma-separated). PRIMARY = android_vr: high-quality DASH, no PO
|
# yt-dlp player clients (comma-separated). PRIMARY = tv_simply: measured 2026-10-10 as the one
|
||||||
# token / JS runtime needed, reliable for normal use. On a bot/DRM/format failure (YouTube's
|
# client that is offered 1080p+ AND finishes the fetch, its stream token minted by the bgutil
|
||||||
# per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
|
# sidecar. android_vr (the primary until then) was still offered 2160p, but every fetch died
|
||||||
# worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
|
# with HTTP 403 — it now needs a stream PO token the sidecar cannot mint — and the fallback
|
||||||
# Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
|
# below served 360p only, so every download came out at 360p without failing. On a
|
||||||
|
# bot/DRM/format failure the worker retries with the FALLBACK set — the POT-capable web clients
|
||||||
|
# (web needs the token + Deno) plus android, whose 360p is the last resort. Both admin-tunable
|
||||||
|
# (env) when YouTube shifts what works.
|
||||||
# A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients
|
# A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients
|
||||||
# of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there.
|
# of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there.
|
||||||
# See adapters.YouTubeAdapter._COOKIE_CAPABLE.
|
# See adapters.YouTubeAdapter._COOKIE_CAPABLE.
|
||||||
download_player_clients: str = "android_vr"
|
download_player_clients: str = "tv_simply"
|
||||||
download_player_clients_fallback: str = "web_safari,web,android"
|
download_player_clients_fallback: str = "web_safari,web,android"
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -342,8 +349,8 @@ class Settings(BaseSettings):
|
|||||||
plex_transcode_crf: int = 20
|
plex_transcode_crf: int = 20
|
||||||
plex_transcode_max_height: int = 1080
|
plex_transcode_max_height: int = 1080
|
||||||
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
|
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
|
||||||
# dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point
|
# dir under files_root. On localdev the files_root is a slow Windows bind-mount, so point
|
||||||
# this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast
|
# this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's files_root is fast
|
||||||
# local disk, so the default is fine there.
|
# local disk, so the default is fine there.
|
||||||
plex_hls_dir: str = ""
|
plex_hls_dir: str = ""
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from sqlalchemy.orm import Session
|
|||||||
from app.discover import cards, queries
|
from app.discover import cards, queries
|
||||||
from app.models import Channel, ComfortPin, PlexItem, PlexShow, Playlist, User, Video
|
from app.models import Channel, ComfortPin, PlexItem, PlexShow, Playlist, User, Video
|
||||||
from app.plex import rollup
|
from app.plex import rollup
|
||||||
from app.plex.cards import leaf_card, show_card
|
from app.plex.cards import SHOW_NEW, leaf_card, show_card
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
|
|
||||||
# kind -> the ComfortPin column that holds it. The client's vocabulary and the schema's, in one place.
|
# kind -> the ComfortPin column that holds it. The client's vocabulary and the schema's, in one place.
|
||||||
@@ -203,8 +203,8 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch, lang: s
|
|||||||
sh = db.get(PlexShow, pin.plex_show_id)
|
sh = db.get(PlexShow, pin.plex_show_id)
|
||||||
if sh is None:
|
if sh is None:
|
||||||
return None
|
return None
|
||||||
status = rollup.state_map(db, user.id, [sh.id]).get(sh.id, "new")
|
state = rollup.state_map(db, user.id, [sh.id]).get(sh.id, SHOW_NEW)
|
||||||
card = cards.from_plex(show_card(sh, status, lang=lang))
|
card = cards.from_plex(show_card(sh, state, lang=lang))
|
||||||
ep = queries.next_episode(db, user.id, sh.id, lang=lang)
|
ep = queries.next_episode(db, user.id, sh.id, lang=lang)
|
||||||
nxt = cards.from_plex(ep) if ep else None
|
nxt = cards.from_plex(ep) if ep else None
|
||||||
|
|
||||||
|
|||||||
@@ -256,16 +256,20 @@ def plex_in_progress(
|
|||||||
if show_libs:
|
if show_libs:
|
||||||
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
||||||
sq = apply_show_state_filter(
|
sq = apply_show_state_filter(
|
||||||
select(PlexShow, prog.c.resumed_at).join(prog, prog.c.show_id == PlexShow.id),
|
select(PlexShow, prog.c.resumed_at, prog.c.total, prog.c.watched, prog.c.inprog).join(
|
||||||
|
prog, prog.c.show_id == PlexShow.id
|
||||||
|
),
|
||||||
prog.c.total,
|
prog.c.total,
|
||||||
prog.c.watched,
|
prog.c.watched,
|
||||||
prog.c.inprog,
|
prog.c.inprog,
|
||||||
"in_progress",
|
"in_progress",
|
||||||
).where(PlexShow.library_id.in_(show_libs))
|
).where(PlexShow.library_id.in_(show_libs))
|
||||||
for sh, resumed_at in db.execute(
|
for sh, resumed_at, total, watched, inprog in db.execute(
|
||||||
sq.order_by(prog.c.resumed_at.desc().nullslast()).limit(limit)
|
sq.order_by(prog.c.resumed_at.desc().nullslast()).limit(limit)
|
||||||
).all():
|
).all():
|
||||||
out.append((resumed_at, show_card(sh, "in_progress", lang=lang)))
|
out.append(
|
||||||
|
(resumed_at, show_card(sh, rollup.show_state(total, watched, inprog), lang=lang))
|
||||||
|
)
|
||||||
|
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|||||||
@@ -106,7 +106,8 @@ class YouTubeAdapter(SourceAdapter):
|
|||||||
# `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
|
# `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
|
||||||
# cookies`, drops it, and the request that is left with no client fails with "Requested format is
|
# cookies`, drops it, and the request that is left with no client fails with "Requested format is
|
||||||
# not available" — which is already a retriable marker above, so the cost is a wasted attempt
|
# not available" — which is already a retriable marker above, so the cost is a wasted attempt
|
||||||
# rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every
|
# rather than a hard failure. Our primary set is tv_simply ALONE, which cannot carry cookies either
|
||||||
|
# (measured on yt-dlp 2026.08.19: `SUPPORTS_COOKIES` False), so without this filter every
|
||||||
# signed-in download would burn its first attempt that way before reaching the POT-backed web
|
# signed-in download would burn its first attempt that way before reaching the POT-backed web
|
||||||
# fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
|
# fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
|
||||||
# ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
|
# ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ def run() -> dict:
|
|||||||
if a.poster_path is None:
|
if a.poster_path is None:
|
||||||
# Record a self-hosted poster for every download (the reliable og:image source):
|
# Record a self-hosted poster for every download (the reliable og:image source):
|
||||||
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
||||||
rel = editmod.ensure_poster(a, settings.download_root)
|
rel = editmod.ensure_poster(a, settings.files_root)
|
||||||
if rel:
|
if rel:
|
||||||
a.poster_path = rel
|
a.poster_path = rel
|
||||||
filled_poster += 1
|
filled_poster += 1
|
||||||
|
|||||||
@@ -0,0 +1,289 @@
|
|||||||
|
"""Ad-hoc clips: a marked range of a YouTube video or a Plex item, cut into a new per-user asset.
|
||||||
|
|
||||||
|
A clip never needs the whole source first. A YouTube clip fetches only its range (yt-dlp's
|
||||||
|
`download_ranges`; measured: 3-7 s whether the video runs four minutes or an hour, and the fetched
|
||||||
|
range starts on the marked frame), and a Plex clip reads the library file in place. Either way the
|
||||||
|
range then goes through ONE re-encode built here, so every clip has the same shape whatever its
|
||||||
|
source: frame-exact at both ends, H.264 at most 1080p (an HDR source tonemapped to SDR — kept as
|
||||||
|
10-bit PQ it played washed out), stereo AAC, in an .mp4 that plays on any phone.
|
||||||
|
|
||||||
|
A clip may also crop the picture to a frame the user drew over the playing video. The frame is
|
||||||
|
given in fractions of the box it was drawn in, with that box's aspect: the YouTube iframe never
|
||||||
|
tells its picture's size, so the frame there is drawn over the player's 16:9 box (a short's 9:16),
|
||||||
|
and the cut maps it onto the real picture (`crop_filter`) — what lies over a letterbox bar is dropped.
|
||||||
|
|
||||||
|
The spec is stored on the job (`DownloadJob.edit_spec`) and is the clip's whole identity: its
|
||||||
|
signature is the asset's cache key, so marking the same range again is an instant cache hit, and a
|
||||||
|
different range or audio track is a different clip.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.downloads.edit import AAC_ARGS, EVEN_FRAME, INPUT_GUARD, X264_ARGS
|
||||||
|
from app.plex.sync import HDR_TRANSFERS
|
||||||
|
|
||||||
|
SOURCES = ("youtube", "plex")
|
||||||
|
# Owner's cap: a clip is a moment, not a way to export a whole film to a public link.
|
||||||
|
MAX_CLIP_S = 600
|
||||||
|
MIN_CLIP_S = 1
|
||||||
|
MAX_HEIGHT = 1080
|
||||||
|
# What yt-dlp fetches the range in (formats.normalize fills the rest): the clip's own ceiling, so the
|
||||||
|
# re-encode never upscales and never throws away a better source than it keeps.
|
||||||
|
YOUTUBE_FETCH_SPEC = {"mode": "av", "max_height": MAX_HEIGHT, "container": "mp4"}
|
||||||
|
# Bump when the spec → output-bytes mapping changes, so a cached clip re-derives.
|
||||||
|
_SIG_VERSION = 1
|
||||||
|
|
||||||
|
_YOUTUBE_ID = re.compile(r"[A-Za-z0-9_-]{11}")
|
||||||
|
_RATING_KEY = re.compile(r"[0-9]{1,32}")
|
||||||
|
_MAX_AUDIO_TRACK = 63
|
||||||
|
# HDR → SDR as measured on the owner's 4K HDR files (2026-10-10): linearise, tonemap (hable),
|
||||||
|
# convert the primaries and transfer to BT.709 and back to 8-bit 4:2:0.
|
||||||
|
_TONEMAP = (
|
||||||
|
"zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=hable:desat=0,"
|
||||||
|
"zscale=t=bt709:m=bt709:r=tv,format=yuv420p"
|
||||||
|
)
|
||||||
|
_SCALE = f"scale=-2:'min({MAX_HEIGHT},ih)'"
|
||||||
|
# A crop frame is at least this share of its box on each side (the player's frame holds the same
|
||||||
|
# floor), and what is left of it on the real picture at least _MIN_VISIBLE of it.
|
||||||
|
MIN_CROP = 0.1
|
||||||
|
_MIN_VISIBLE = 0.05
|
||||||
|
# The box aspects a frame may be drawn in: a 9:16 short to a 21:9 film, with room either side.
|
||||||
|
_CROP_ASPECTS = (0.25, 4.0)
|
||||||
|
# Float slack on the frame's far edges (x + w, y + h), which the client rounds to 4 decimals.
|
||||||
|
_EDGE_SLACK = 1e-3
|
||||||
|
|
||||||
|
|
||||||
|
class ClipError(Exception):
|
||||||
|
"""A clip request that cannot be cut; `reason` is an i18n-mappable key, `params` the numbers its
|
||||||
|
message names (the cap itself, so the client never says a stale one)."""
|
||||||
|
|
||||||
|
def __init__(self, reason: str, **params: float):
|
||||||
|
self.reason = reason
|
||||||
|
self.params = params
|
||||||
|
super().__init__(reason)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(
|
||||||
|
source: str,
|
||||||
|
ref: str,
|
||||||
|
start_s: float,
|
||||||
|
end_s: float,
|
||||||
|
audio_track: int | None = None,
|
||||||
|
crop: dict | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""The validated, canonical clip spec. Times are rounded to the millisecond and a crop to four
|
||||||
|
decimals, so the same marks always give the same cache key. Raises ClipError."""
|
||||||
|
if source not in SOURCES:
|
||||||
|
raise ClipError("bad_source")
|
||||||
|
pattern = _YOUTUBE_ID if source == "youtube" else _RATING_KEY
|
||||||
|
if not isinstance(ref, str) or not pattern.fullmatch(ref):
|
||||||
|
raise ClipError("bad_source")
|
||||||
|
if not (math.isfinite(start_s) and math.isfinite(end_s)) or start_s < 0:
|
||||||
|
raise ClipError("bad_range")
|
||||||
|
start, end = round(start_s, 3), round(end_s, 3)
|
||||||
|
if end - start < MIN_CLIP_S:
|
||||||
|
raise ClipError("too_short")
|
||||||
|
if end - start > MAX_CLIP_S:
|
||||||
|
raise ClipError("too_long", limit_s=MAX_CLIP_S)
|
||||||
|
spec: dict = {"source": source, "ref": ref, "start_s": start, "end_s": end}
|
||||||
|
if source == "plex" and audio_track is not None:
|
||||||
|
if not 0 <= audio_track <= _MAX_AUDIO_TRACK:
|
||||||
|
raise ClipError("bad_audio")
|
||||||
|
spec["audio_track"] = audio_track
|
||||||
|
frame = _crop(crop)
|
||||||
|
if frame is not None:
|
||||||
|
spec["crop"] = frame
|
||||||
|
return spec
|
||||||
|
|
||||||
|
|
||||||
|
def _crop(raw: dict | None) -> dict | None:
|
||||||
|
"""A crop frame, canonical — or None for no crop, which a frame over the whole box also is (the
|
||||||
|
same clip as one cut without a frame, so the same cache key)."""
|
||||||
|
if raw is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
x, y, w, h, aspect = (float(raw[k]) for k in ("x", "y", "w", "h", "aspect"))
|
||||||
|
except (KeyError, TypeError, ValueError) as e:
|
||||||
|
raise ClipError("bad_crop") from e
|
||||||
|
if not all(math.isfinite(v) for v in (x, y, w, h, aspect)):
|
||||||
|
raise ClipError("bad_crop")
|
||||||
|
lo, hi = _CROP_ASPECTS
|
||||||
|
if not lo <= aspect <= hi or x < 0 or y < 0:
|
||||||
|
raise ClipError("bad_crop")
|
||||||
|
if w < MIN_CROP - _EDGE_SLACK or h < MIN_CROP - _EDGE_SLACK:
|
||||||
|
raise ClipError("bad_crop")
|
||||||
|
if x + w > 1 + _EDGE_SLACK or y + h > 1 + _EDGE_SLACK:
|
||||||
|
raise ClipError("bad_crop")
|
||||||
|
frame = {"x": x, "y": y, "w": min(w, 1 - x), "h": min(h, 1 - y), "aspect": aspect}
|
||||||
|
frame = {k: round(v, 4) for k, v in frame.items()}
|
||||||
|
if (frame["x"], frame["y"], frame["w"], frame["h"]) == (0, 0, 1, 1):
|
||||||
|
return None
|
||||||
|
return frame
|
||||||
|
|
||||||
|
|
||||||
|
def duration(spec: dict) -> float:
|
||||||
|
return spec["end_s"] - spec["start_s"]
|
||||||
|
|
||||||
|
|
||||||
|
def sig(spec: dict) -> str:
|
||||||
|
"""The clip's cache signature: the spec itself, versioned."""
|
||||||
|
body = json.dumps({**spec, "_v": _SIG_VERSION}, sort_keys=True, separators=(",", ":"))
|
||||||
|
return hashlib.sha256(body.encode()).hexdigest()[:24]
|
||||||
|
|
||||||
|
|
||||||
|
def asset_ref(user_id: int, spec: dict) -> str:
|
||||||
|
"""The asset's `source_ref`. The user id makes a clip private to whoever cut it, like an edit:
|
||||||
|
never dedup-shared, never cache-served to someone else."""
|
||||||
|
return f"{user_id}:{spec['source']}:{spec['ref']}"
|
||||||
|
|
||||||
|
|
||||||
|
def reference_url(spec: dict) -> str | None:
|
||||||
|
"""Where a YouTube clip came from: its video at the clip's start. A Plex clip has no public page."""
|
||||||
|
if spec.get("source") != "youtube" or not spec.get("ref"):
|
||||||
|
return None
|
||||||
|
return f"https://www.youtube.com/watch?v={spec['ref']}&t={int(spec.get('start_s') or 0)}s"
|
||||||
|
|
||||||
|
|
||||||
|
def is_hdr(probe: dict) -> bool:
|
||||||
|
"""Whether the first video stream of an ffprobe `-show_streams` JSON carries an HDR transfer."""
|
||||||
|
for s in probe.get("streams") or []:
|
||||||
|
if s.get("codec_type") == "video":
|
||||||
|
# PQ / HLG: an 8-bit SDR encode must tonemap them (the one set the Plex stream uses).
|
||||||
|
return s.get("color_transfer") in HDR_TRANSFERS
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def display_aspect(probe: dict) -> float | None:
|
||||||
|
"""The picture's shape as a player shows it, from an ffprobe `-show_streams` JSON: the first video
|
||||||
|
stream's width × its sample aspect ratio over its height, turned a quarter when the stream says
|
||||||
|
it is rotated (ffmpeg turns such a stream upright before any filter of ours, so a crop is cut
|
||||||
|
from the upright picture). None when the stream has no size."""
|
||||||
|
for s in probe.get("streams") or []:
|
||||||
|
if s.get("codec_type") != "video":
|
||||||
|
continue
|
||||||
|
w, h = s.get("width"), s.get("height")
|
||||||
|
if not isinstance(w, int) or not isinstance(h, int) or w <= 0 or h <= 0:
|
||||||
|
return None
|
||||||
|
dar = w * (_ratio(s.get("sample_aspect_ratio")) or 1.0) / h
|
||||||
|
return 1 / dar if _quarter_turned(s) else dar
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _ratio(v: object) -> float | None:
|
||||||
|
"""'32:27' → 1.185…; '0:1', 'N/A', a missing or malformed value → None."""
|
||||||
|
if not isinstance(v, str) or ":" not in v:
|
||||||
|
return None
|
||||||
|
num, _, den = v.partition(":")
|
||||||
|
try:
|
||||||
|
n, d = float(num), float(den)
|
||||||
|
except ValueError: # silent-ok: a malformed ratio reads as none given; the caller takes 1:1
|
||||||
|
return None
|
||||||
|
return n / d if n > 0 and d > 0 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _quarter_turned(stream: dict) -> bool:
|
||||||
|
rotation: object = (stream.get("tags") or {}).get("rotate")
|
||||||
|
for sd in stream.get("side_data_list") or []:
|
||||||
|
if "rotation" in sd:
|
||||||
|
rotation = sd["rotation"]
|
||||||
|
try:
|
||||||
|
return round(float(str(rotation))) % 180 == 90
|
||||||
|
except ValueError: # silent-ok: no (None) or a malformed rotation: the stream is upright
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def crop_filter(crop: dict, dar: float) -> str | None:
|
||||||
|
"""The ffmpeg `crop` for a frame drawn in a box of `crop["aspect"]`, over a picture of display
|
||||||
|
aspect `dar` fitted into that box as a player fits it (whole, centred, bars on the short sides).
|
||||||
|
The frame's part over a bar is dropped: "" when what is left is the whole picture, None when too
|
||||||
|
little of the frame lies on the picture at all.
|
||||||
|
|
||||||
|
In fractions of the input's own width and height, so a sample aspect ratio other than 1:1 (an
|
||||||
|
anamorphic DVD) needs nothing of its own, and floored to even pixels: an even 4:2:0 frame, and a
|
||||||
|
window that never runs past the picture's far edge."""
|
||||||
|
box = crop["aspect"]
|
||||||
|
pic_w, pic_h = (1.0, box / dar) if dar > box else (dar / box, 1.0)
|
||||||
|
|
||||||
|
def span(start: float, size: float, extent: float) -> tuple[float, float]:
|
||||||
|
origin = (1 - extent) / 2
|
||||||
|
a = min(1.0, max(0.0, (start - origin) / extent))
|
||||||
|
b = min(1.0, max(0.0, (start + size - origin) / extent))
|
||||||
|
return a, b - a
|
||||||
|
|
||||||
|
x, w = span(crop["x"], crop["w"], pic_w)
|
||||||
|
y, h = span(crop["y"], crop["h"], pic_h)
|
||||||
|
if w < _MIN_VISIBLE or h < _MIN_VISIBLE:
|
||||||
|
return None
|
||||||
|
if w > 1 - 1e-4 and h > 1 - 1e-4:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def px(axis: str, f: float) -> str:
|
||||||
|
return f"trunc({axis}*{f:.6f}/2)*2"
|
||||||
|
|
||||||
|
return f"crop={px('iw', w)}:{px('ih', h)}:{px('iw', x)}:{px('ih', y)}"
|
||||||
|
|
||||||
|
|
||||||
|
def audio_track_count(probe: dict) -> int:
|
||||||
|
return sum(1 for s in probe.get("streams") or [] if s.get("codec_type") == "audio")
|
||||||
|
|
||||||
|
|
||||||
|
def build_cmd(
|
||||||
|
src: Path,
|
||||||
|
dest: Path,
|
||||||
|
start_s: float,
|
||||||
|
dur_s: float,
|
||||||
|
*,
|
||||||
|
audio_track: int | None,
|
||||||
|
hdr: bool,
|
||||||
|
threads: int,
|
||||||
|
crop: str = "",
|
||||||
|
) -> list[str]:
|
||||||
|
"""ffmpeg for one clip: an input seek (decodes from the keyframe before `start_s` and drops the
|
||||||
|
frames up to it, so the clip starts on the marked frame), the first video stream, the chosen audio
|
||||||
|
track (the first when none was chosen; none at all is fine), no subtitles, data or chapters.
|
||||||
|
A `crop` (crop_filter) comes first: the tonemap and the scale then work on the frame it keeps."""
|
||||||
|
vf = ",".join([*([crop] if crop else []), *([_TONEMAP] if hdr else []), _SCALE, EVEN_FRAME])
|
||||||
|
audio = f"0:a:{audio_track}" if audio_track is not None else "0:a:0?"
|
||||||
|
return [
|
||||||
|
"ffmpeg",
|
||||||
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-progress",
|
||||||
|
"pipe:1",
|
||||||
|
"-nostats",
|
||||||
|
"-ss",
|
||||||
|
f"{start_s:.3f}",
|
||||||
|
*INPUT_GUARD,
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
|
"-t",
|
||||||
|
f"{dur_s:.3f}",
|
||||||
|
"-map",
|
||||||
|
"0:v:0",
|
||||||
|
"-map",
|
||||||
|
audio,
|
||||||
|
"-sn",
|
||||||
|
"-dn",
|
||||||
|
# A source's chapters would ride into the .mp4 as a data track of their own (an mkv's
|
||||||
|
# chapter list came out as `bin_data`); a clip of a moment has no chapters to keep.
|
||||||
|
"-map_chapters",
|
||||||
|
"-1",
|
||||||
|
"-vf",
|
||||||
|
vf,
|
||||||
|
*X264_ARGS,
|
||||||
|
"-threads",
|
||||||
|
str(threads),
|
||||||
|
*AAC_ARGS,
|
||||||
|
"-ac",
|
||||||
|
"2",
|
||||||
|
"-movflags",
|
||||||
|
"+faststart",
|
||||||
|
str(dest),
|
||||||
|
]
|
||||||
@@ -30,6 +30,7 @@ from sqlalchemy import select
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs
|
from app import logs
|
||||||
|
from app.downloads import clip as clipmod
|
||||||
from app.downloads import formats
|
from app.downloads import formats
|
||||||
from app.downloads.adapters import _host_is
|
from app.downloads.adapters import _host_is
|
||||||
from app.models import DownloadCredential, MediaAsset
|
from app.models import DownloadCredential, MediaAsset
|
||||||
@@ -259,6 +260,28 @@ def usable_cookie(
|
|||||||
return None
|
return None
|
||||||
if not is_authed_asset(asset, spec, user_id, url):
|
if not is_authed_asset(asset, spec, user_id, url):
|
||||||
return None # shared asset — never attach a cookie to a non-private identity
|
return None # shared asset — never attach a cookie to a non-private identity
|
||||||
|
return _readable(cred, user_id, host_key)
|
||||||
|
|
||||||
|
|
||||||
|
def usable_clip_cookie(
|
||||||
|
db: Session, user_id: int, url: str | None, asset: MediaAsset, clip_spec: dict
|
||||||
|
) -> str | None:
|
||||||
|
"""`usable_cookie` for an ad-hoc clip's range fetch. The second gate is the clip's own identity:
|
||||||
|
a clip asset is private by construction (`clip.asset_ref` names its user, see downloads/clip),
|
||||||
|
so a cookie goes with it exactly when that asset is THIS user's clip of `clip_spec`. Without it a
|
||||||
|
video YouTube gates signed out could never be clipped, while its download goes through."""
|
||||||
|
host_key = host_key_for_url(url)
|
||||||
|
if host_key is None:
|
||||||
|
return None
|
||||||
|
cred = get_credential(db, user_id, host_key)
|
||||||
|
if cred is None:
|
||||||
|
return None
|
||||||
|
if asset.source_kind != "clip" or asset.source_ref != clipmod.asset_ref(user_id, clip_spec):
|
||||||
|
return None
|
||||||
|
return _readable(cred, user_id, host_key)
|
||||||
|
|
||||||
|
|
||||||
|
def _readable(cred: DownloadCredential, user_id: int, host_key: str) -> str | None:
|
||||||
raw = decrypt_optional(cred.cookies_enc)
|
raw = decrypt_optional(cred.cookies_enc)
|
||||||
if not raw:
|
if not raw:
|
||||||
log.warning(
|
log.warning(
|
||||||
@@ -284,6 +307,14 @@ def resolve_cookiefile(
|
|||||||
return _write_cookiefile(raw, staging) if raw else None
|
return _write_cookiefile(raw, staging) if raw else None
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_clip_cookiefile(
|
||||||
|
db: Session, user_id: int, url: str | None, asset: MediaAsset, clip_spec: dict, staging: Path
|
||||||
|
) -> Path | None:
|
||||||
|
"""`resolve_cookiefile` for an ad-hoc clip (`usable_clip_cookie`)."""
|
||||||
|
raw = usable_clip_cookie(db, user_id, url, asset, clip_spec)
|
||||||
|
return _write_cookiefile(raw, staging) if raw else None
|
||||||
|
|
||||||
|
|
||||||
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
||||||
"""Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path.
|
"""Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path.
|
||||||
|
|
||||||
|
|||||||
+75
-117
@@ -188,6 +188,51 @@ def clip_duration(spec: dict, source_duration: int | None) -> int | None:
|
|||||||
|
|
||||||
# --- ffmpeg: trim / crop --------------------------------------------------------------------
|
# --- ffmpeg: trim / crop --------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Every libx264 / aac re-encode in this package uses these, so a re-encoded file is always the
|
||||||
|
# phone-safe shape (`playable`): `-pix_fmt yuv420p` pins 8-bit 4:2:0 — without it libx264 keeps a
|
||||||
|
# 10-bit or 4:4:4 source's pixel format and writes H.264 High 10 / High 4:4:4, which browsers decode
|
||||||
|
# only patchily (the one safe H.264 is 8-bit 4:2:0).
|
||||||
|
X264_ARGS = ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p"]
|
||||||
|
AAC_ARGS = ["-c:a", "aac", "-b:a", "192k"]
|
||||||
|
# The last step of every video filter chain a libx264 re-encode runs: 4:2:0 halves the chroma, so
|
||||||
|
# libx264 refuses an odd width or height ("width not divisible by 2") — measured on an odd source
|
||||||
|
# that is not 4:2:0 already (a 4:4:4 or RGB screen capture, 1365x767). A crop of a 4:2:0 source is
|
||||||
|
# rounded to its chroma grid by ffmpeg itself; this holds every other path to the encoder to the same
|
||||||
|
# even frame. Rounding down to even costs at most one pixel row/column.
|
||||||
|
EVEN_FRAME = "scale=trunc(iw/2)*2:trunc(ih/2)*2"
|
||||||
|
|
||||||
|
# The demuxers ffmpeg/ffprobe may use to open a library file — single-file containers only. Without
|
||||||
|
# it ffmpeg picks the demuxer from the file's CONTENT and NAME, and an upload's name is the user's:
|
||||||
|
# a playlist-type demuxer (HLS, DASH, concat) opens whatever local paths its entries name, so a
|
||||||
|
# crafted upload could copy another user's clip or a Plex file into its own converted asset. A file
|
||||||
|
# no demuxer here can open fails the probe, and an unprobeable upload is a download-only 'file'.
|
||||||
|
MEDIA_DEMUXERS = (
|
||||||
|
"mov", # mp4, m4a, mov, 3gp
|
||||||
|
"matroska", # mkv, webm
|
||||||
|
"mpegts",
|
||||||
|
"mpeg",
|
||||||
|
"avi",
|
||||||
|
"asf",
|
||||||
|
"flv",
|
||||||
|
"ogg",
|
||||||
|
"mp3",
|
||||||
|
"aac",
|
||||||
|
"ac3",
|
||||||
|
"eac3",
|
||||||
|
"flac",
|
||||||
|
"wav",
|
||||||
|
"w64",
|
||||||
|
"aiff",
|
||||||
|
"caf",
|
||||||
|
"amr",
|
||||||
|
)
|
||||||
|
INPUT_GUARD = ["-format_whitelist", ",".join(MEDIA_DEMUXERS)]
|
||||||
|
# A downloaded thumbnail is the remote site's bytes under the suffix yt-dlp gave it (`.png`, `.webp`,
|
||||||
|
# `.jpg` — `pipeline._THUMB_SUFFIXES`): only the single-image demuxers those open, so a "thumbnail"
|
||||||
|
# that is really a playlist cannot pull a local file into the poster. Anything else fails, and the
|
||||||
|
# poster falls back to a cut frame (`ensure_poster`).
|
||||||
|
THUMB_GUARD = ["-format_whitelist", "png_pipe,jpeg_pipe,webp_pipe"]
|
||||||
|
|
||||||
|
|
||||||
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
||||||
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
|
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
|
||||||
@@ -214,7 +259,7 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
|||||||
]
|
]
|
||||||
if start:
|
if start:
|
||||||
cmd += ["-ss", f"{start:.3f}"]
|
cmd += ["-ss", f"{start:.3f}"]
|
||||||
cmd += ["-i", str(src)]
|
cmd += [*INPUT_GUARD, "-i", str(src)]
|
||||||
if end is not None:
|
if end is not None:
|
||||||
dur = end - start
|
dur = end - start
|
||||||
if dur > 0:
|
if dur > 0:
|
||||||
@@ -222,21 +267,9 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
|||||||
|
|
||||||
if reencode:
|
if reencode:
|
||||||
# Input -ss + re-encode is frame-accurate (decodes from the prior keyframe, discards up to
|
# Input -ss + re-encode is frame-accurate (decodes from the prior keyframe, discards up to
|
||||||
# the exact cut). A crop adds the filter; an accurate trim re-encodes with no filter.
|
# the exact cut). A crop adds the filter; an accurate trim only evens the frame.
|
||||||
if crop:
|
cropf = f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}," if crop else ""
|
||||||
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
|
cmd += ["-vf", cropf + EVEN_FRAME, *X264_ARGS, *AAC_ARGS]
|
||||||
cmd += [
|
|
||||||
"-c:v",
|
|
||||||
"libx264",
|
|
||||||
"-preset",
|
|
||||||
"veryfast",
|
|
||||||
"-crf",
|
|
||||||
"20",
|
|
||||||
"-c:a",
|
|
||||||
"aac",
|
|
||||||
"-b:a",
|
|
||||||
"192k",
|
|
||||||
]
|
|
||||||
else:
|
else:
|
||||||
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
||||||
|
|
||||||
@@ -284,11 +317,14 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
|||||||
parts, labels = [], []
|
parts, labels = [], []
|
||||||
for i, s in enumerate(segs):
|
for i, s in enumerate(segs):
|
||||||
st, en = s["start_s"], s["end_s"]
|
st, en = s["start_s"], s["end_s"]
|
||||||
parts.append(f"[0:v]trim=start={st}:end={en},setpts=PTS-STARTPTS{cropf}[v{i}]")
|
parts.append(
|
||||||
|
f"[0:v]trim=start={st}:end={en},setpts=PTS-STARTPTS{cropf},{EVEN_FRAME}[v{i}]"
|
||||||
|
)
|
||||||
parts.append(f"[0:a]atrim=start={st}:end={en},asetpts=PTS-STARTPTS[a{i}]")
|
parts.append(f"[0:a]atrim=start={st}:end={en},asetpts=PTS-STARTPTS[a{i}]")
|
||||||
labels.append(f"[v{i}][a{i}]")
|
labels.append(f"[v{i}][a{i}]")
|
||||||
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
||||||
cmd = head + [
|
cmd = head + [
|
||||||
|
*INPUT_GUARD,
|
||||||
"-i",
|
"-i",
|
||||||
str(src),
|
str(src),
|
||||||
"-filter_complex",
|
"-filter_complex",
|
||||||
@@ -297,26 +333,23 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
|||||||
"[v]",
|
"[v]",
|
||||||
"-map",
|
"-map",
|
||||||
"[a]",
|
"[a]",
|
||||||
"-c:v",
|
*X264_ARGS,
|
||||||
"libx264",
|
*AAC_ARGS,
|
||||||
"-preset",
|
|
||||||
"veryfast",
|
|
||||||
"-crf",
|
|
||||||
"20",
|
|
||||||
"-c:a",
|
|
||||||
"aac",
|
|
||||||
"-b:a",
|
|
||||||
"192k",
|
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
listp = staging / "concat.txt"
|
listp = staging / "concat.txt"
|
||||||
lines = []
|
lines = ["ffconcat version 1.0"]
|
||||||
for s in segs:
|
for s in segs:
|
||||||
lines.append(f"file '{_concat_quote(src)}'")
|
lines.append(f"file '{_concat_quote(src)}'")
|
||||||
|
# ffmpeg hands the list's whitelist (which must name `concat` to read the list) down to
|
||||||
|
# the file it names, so a library file whose bytes are a concat list would be read as one
|
||||||
|
# and open the paths inside it. Each entry's own whitelist replaces the inherited one.
|
||||||
|
lines.append(f"option format_whitelist {INPUT_GUARD[1]}")
|
||||||
lines.append(f"inpoint {s['start_s']}")
|
lines.append(f"inpoint {s['start_s']}")
|
||||||
lines.append(f"outpoint {s['end_s']}")
|
lines.append(f"outpoint {s['end_s']}")
|
||||||
prep[listp] = "\n".join(lines) + "\n"
|
prep[listp] = "\n".join(lines) + "\n"
|
||||||
cmd = head + ["-f", "concat", "-safe", "0", "-i", str(listp), "-c", "copy"]
|
guard = ["-format_whitelist", "concat," + INPUT_GUARD[1]]
|
||||||
|
cmd = head + ["-f", "concat", "-safe", "0", *guard, "-i", str(listp), "-c", "copy"]
|
||||||
|
|
||||||
if out_ext in ("mp4", "m4a", "mov"):
|
if out_ext in ("mp4", "m4a", "mov"):
|
||||||
cmd += ["-movflags", "+faststart"]
|
cmd += ["-movflags", "+faststart"]
|
||||||
@@ -474,14 +507,7 @@ def run_ffmpeg(
|
|||||||
raise StatedFailure(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
raise StatedFailure(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||||
|
|
||||||
|
|
||||||
# --- ffmpeg: universal browser playability --------------------------------------------------
|
# --- ffprobe ---------------------------------------------------------------------------------
|
||||||
# A downloaded file must play in EVERY browser, whatever the source shipped. Universal = H.264
|
|
||||||
# video + AAC/MP3 audio in an mp4 container: H.264 is the only video codec all browsers decode
|
|
||||||
# (VP9/AV1 fail on Safari/iOS; HEVC fails on Chrome/Firefox), and mp4/AAC is the safe pairing.
|
|
||||||
# Anything else is fixed post-download: re-encode the offending stream(s), or just remux when the
|
|
||||||
# codecs are already fine but the container isn't.
|
|
||||||
_BROWSER_SAFE_VCODEC = ("h264",)
|
|
||||||
_BROWSER_SAFE_ACODEC = ("aac", "mp3")
|
|
||||||
|
|
||||||
|
|
||||||
def run_ffprobe_json(
|
def run_ffprobe_json(
|
||||||
@@ -493,10 +519,10 @@ def run_ffprobe_json(
|
|||||||
) -> dict | None:
|
) -> dict | None:
|
||||||
"""One ffprobe call → its parsed JSON dict, or None on any failure (timeout, non-zero exit,
|
"""One ffprobe call → its parsed JSON dict, or None on any failure (timeout, non-zero exit,
|
||||||
missing binary, unparseable output). The SINGLE ffprobe-JSON-invocation point in the downloads
|
missing binary, unparseable output). The SINGLE ffprobe-JSON-invocation point in the downloads
|
||||||
package: the browser-playability codec probe here and the shared media probe (`probe.probe_media`,
|
package: the shared media probe (`probe.probe_json`: the stored facts and the phone-safe rule) goes
|
||||||
used by both the download worker's finalize and the upload ingest) both go through it, so the
|
through it, so the
|
||||||
subprocess flags/timeout/error handling live in one place instead of drifting copies."""
|
subprocess flags/timeout/error handling live in one place instead of drifting copies."""
|
||||||
args = ["ffprobe", "-v", "error"]
|
args = ["ffprobe", "-v", "error", *INPUT_GUARD]
|
||||||
if show_streams:
|
if show_streams:
|
||||||
args.append("-show_streams")
|
args.append("-show_streams")
|
||||||
if show_format:
|
if show_format:
|
||||||
@@ -517,35 +543,6 @@ def run_ffprobe_json(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _codecs_from_streams(streams: list) -> tuple[str | None, str | None]:
|
|
||||||
"""(video_codec, audio_codec) lowercased from an ffprobe JSON stream list — the first of each kind.
|
|
||||||
|
|
||||||
Parsed from the JSON `streams` list, NOT ffprobe's flat `default=nw=1` output: there the fields
|
|
||||||
arrive in ffprobe's own struct order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line
|
|
||||||
"remember the last codec_type" parser attributes every name to the PREVIOUS stream. Measured on real
|
|
||||||
downloads: an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
|
|
||||||
believed the video stream was AAC and re-encoded a perfectly fine file on every download. Reading each
|
|
||||||
stream dict by its own `codec_type` is immune to that ordering."""
|
|
||||||
vcodec = acodec = None
|
|
||||||
for s in streams:
|
|
||||||
name = (s.get("codec_name") or "").lower() or None
|
|
||||||
ctype = (s.get("codec_type") or "").lower()
|
|
||||||
if ctype == "video" and vcodec is None:
|
|
||||||
vcodec = name
|
|
||||||
elif ctype == "audio" and acodec is None:
|
|
||||||
acodec = name
|
|
||||||
return vcodec, acodec
|
|
||||||
|
|
||||||
|
|
||||||
def probe_codecs(path: Path) -> tuple[str | None, str | None]:
|
|
||||||
"""Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing
|
|
||||||
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
|
|
||||||
data = run_ffprobe_json(path, entries="stream=codec_type,codec_name")
|
|
||||||
if data is None:
|
|
||||||
return None, None
|
|
||||||
return _codecs_from_streams(data.get("streams") or [])
|
|
||||||
|
|
||||||
|
|
||||||
def probe_duration(path: Path) -> int:
|
def probe_duration(path: Path) -> int:
|
||||||
"""Media duration in whole seconds via ffprobe, or 0 if unknown — a progress denominator for a
|
"""Media duration in whole seconds via ffprobe, or 0 if unknown — a progress denominator for a
|
||||||
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
|
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
|
||||||
@@ -555,6 +552,7 @@ def probe_duration(path: Path) -> int:
|
|||||||
"ffprobe",
|
"ffprobe",
|
||||||
"-v",
|
"-v",
|
||||||
"error",
|
"error",
|
||||||
|
*INPUT_GUARD,
|
||||||
"-show_entries",
|
"-show_entries",
|
||||||
"format=duration",
|
"format=duration",
|
||||||
"-of",
|
"-of",
|
||||||
@@ -577,49 +575,6 @@ def probe_duration(path: Path) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def browser_transcode_flags(
|
|
||||||
vcodec: str | None, acodec: str | None, is_mp4: bool
|
|
||||||
) -> tuple[bool, bool, bool]:
|
|
||||||
"""Decide what (if anything) makes a file universally browser-playable. Returns
|
|
||||||
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
|
|
||||||
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
|
|
||||||
rv = bool(vcodec) and vcodec not in _BROWSER_SAFE_VCODEC
|
|
||||||
ra = bool(acodec) and acodec not in _BROWSER_SAFE_ACODEC
|
|
||||||
return (rv or ra or not is_mp4), rv, ra
|
|
||||||
|
|
||||||
|
|
||||||
def build_browser_safe_cmd(
|
|
||||||
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool, threads: int | None = None
|
|
||||||
) -> list[str]:
|
|
||||||
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
|
||||||
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
|
||||||
pipe:1` streams machine-readable progress for run_ffmpeg. `threads` caps the encoder so a
|
|
||||||
re-encode can't claim every core on the shared box (see the worker-child de-prioritization)."""
|
|
||||||
cmd = [
|
|
||||||
"ffmpeg",
|
|
||||||
"-y",
|
|
||||||
"-hide_banner",
|
|
||||||
"-loglevel",
|
|
||||||
"error",
|
|
||||||
"-nostdin",
|
|
||||||
"-progress",
|
|
||||||
"pipe:1",
|
|
||||||
"-nostats",
|
|
||||||
"-i",
|
|
||||||
str(src),
|
|
||||||
]
|
|
||||||
cmd += (
|
|
||||||
["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"]
|
|
||||||
if reencode_video
|
|
||||||
else ["-c:v", "copy"]
|
|
||||||
)
|
|
||||||
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
|
||||||
if threads and threads > 0:
|
|
||||||
cmd += ["-threads", str(threads)]
|
|
||||||
cmd += ["-movflags", "+faststart", str(dest)]
|
|
||||||
return cmd
|
|
||||||
|
|
||||||
|
|
||||||
# --- filmstrip (editor scrub track) ---------------------------------------------------------
|
# --- filmstrip (editor scrub track) ---------------------------------------------------------
|
||||||
|
|
||||||
_SB_COLS = 12
|
_SB_COLS = 12
|
||||||
@@ -655,6 +610,7 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
|
|||||||
"-loglevel",
|
"-loglevel",
|
||||||
"error",
|
"error",
|
||||||
"-nostdin",
|
"-nostdin",
|
||||||
|
*INPUT_GUARD,
|
||||||
"-i",
|
"-i",
|
||||||
str(src),
|
str(src),
|
||||||
"-frames:v",
|
"-frames:v",
|
||||||
@@ -678,6 +634,7 @@ def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
|||||||
"-nostdin",
|
"-nostdin",
|
||||||
"-ss",
|
"-ss",
|
||||||
f"{max(0.0, at_s):.3f}",
|
f"{max(0.0, at_s):.3f}",
|
||||||
|
*INPUT_GUARD,
|
||||||
"-i",
|
"-i",
|
||||||
str(src),
|
str(src),
|
||||||
"-frames:v",
|
"-frames:v",
|
||||||
@@ -699,6 +656,7 @@ def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
|
|||||||
"-loglevel",
|
"-loglevel",
|
||||||
"error",
|
"error",
|
||||||
"-nostdin",
|
"-nostdin",
|
||||||
|
*THUMB_GUARD,
|
||||||
"-i",
|
"-i",
|
||||||
str(src),
|
str(src),
|
||||||
"-frames:v",
|
"-frames:v",
|
||||||
@@ -734,7 +692,7 @@ def _step_failure(err: BaseException) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _is_jpeg(path: Path) -> bool:
|
def _is_jpeg(path: Path) -> bool:
|
||||||
"""True if the file starts with the JPEG SOI marker (FF D8 FF)."""
|
"""True if the file starts with the JPEG SOI marker (0xFF 0xD8 0xFF)."""
|
||||||
try:
|
try:
|
||||||
with path.open("rb") as f:
|
with path.open("rb") as f:
|
||||||
return f.read(3) == b"\xff\xd8\xff"
|
return f.read(3) == b"\xff\xd8\xff"
|
||||||
@@ -774,7 +732,7 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
|||||||
return dest if dest.exists() else None
|
return dest if dest.exists() else None
|
||||||
|
|
||||||
|
|
||||||
def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
def ensure_poster(asset, files_root: str, timeout_s: int = 30) -> str | None:
|
||||||
"""Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL
|
"""Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL
|
||||||
like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's
|
like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's
|
||||||
`<base>.jpg` sidecar — so Plex uses it too and `delete_asset_files` cleans it up — and returned
|
`<base>.jpg` sidecar — so Plex uses it too and `delete_asset_files` cleans it up — and returned
|
||||||
@@ -782,7 +740,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
|||||||
short clip) dodges black intro frames."""
|
short clip) dodges black intro frames."""
|
||||||
if not asset.rel_path:
|
if not asset.rel_path:
|
||||||
return None
|
return None
|
||||||
root = Path(download_root)
|
root = Path(files_root)
|
||||||
src = root / asset.rel_path
|
src = root / asset.rel_path
|
||||||
if not src.exists():
|
if not src.exists():
|
||||||
return None
|
return None
|
||||||
@@ -798,7 +756,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
|||||||
return rel if dest.exists() else None
|
return rel if dest.exists() else None
|
||||||
|
|
||||||
|
|
||||||
def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | None:
|
def ensure_storyboard(asset, files_root: str, timeout_s: int = 90) -> dict | None:
|
||||||
"""Return the filmstrip meta for a ready source asset, generating the sprite on first use.
|
"""Return the filmstrip meta for a ready source asset, generating the sprite on first use.
|
||||||
|
|
||||||
Best-effort + time-bounded: for a very long/high-res source the one-pass decode can be slow,
|
Best-effort + time-bounded: for a very long/high-res source the one-pass decode can be slow,
|
||||||
@@ -810,7 +768,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict |
|
|||||||
if not asset.rel_path:
|
if not asset.rel_path:
|
||||||
return None
|
return None
|
||||||
geom = storyboard_geometry(asset.duration_s)
|
geom = storyboard_geometry(asset.duration_s)
|
||||||
root = Path(download_root)
|
root = Path(files_root)
|
||||||
rel = f".storyboards/asset-{asset.id}.jpg"
|
rel = f".storyboards/asset-{asset.id}.jpg"
|
||||||
dest = root / rel
|
dest = root / rel
|
||||||
|
|
||||||
|
|||||||
@@ -37,9 +37,10 @@ from app.downloads.credentials import host_key_for_url
|
|||||||
|
|
||||||
# Worker-authored strings that this module owns because each is coupled to `_RULES`.
|
# Worker-authored strings that this module owns because each is coupled to `_RULES`.
|
||||||
#
|
#
|
||||||
# QUOTA_FULL_MESSAGE is raised by the worker and must classify back to `quota_exceeded`; its rule
|
# QUOTA_FULL_MESSAGE and the three *_TOO_BIG_MESSAGEs are raised by the worker and must classify
|
||||||
# below matches a fragment of it, and test_download_errors pins the pair, because a message and its
|
# back to `quota_exceeded`; their rules below match a fragment of each, and test_download_errors pins
|
||||||
# rule drifting apart is exactly how a category silently becomes unreachable.
|
# each one, because a message and its rule drifting apart is exactly how a
|
||||||
|
# category silently becomes unreachable.
|
||||||
#
|
#
|
||||||
# NETWORK_REDACTED_MESSAGE REPLACES a message after classification (`user_error_text`). It is written
|
# NETWORK_REDACTED_MESSAGE REPLACES a message after classification (`user_error_text`). It is written
|
||||||
# to the shared asset too, and a second job that inherits that failed asset re-diagnoses the stored
|
# to the shared asset too, and a second job that inherits that failed asset re-diagnoses the stored
|
||||||
@@ -53,6 +54,20 @@ from app.downloads.credentials import host_key_for_url
|
|||||||
# constants, imported above; each has a rule below that gives its category back, and a catalog entry
|
# constants, imported above; each has a rule below that gives its category back, and a catalog entry
|
||||||
# that gives the site's sentence.
|
# that gives the site's sentence.
|
||||||
QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
|
QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
|
||||||
|
# A download, a cut clip, or an upload's converted file, held to the space left, which is not zero:
|
||||||
|
# "full" would be untrue. All three classify by the one phrase they share.
|
||||||
|
DOWNLOAD_TOO_BIG_MESSAGE = (
|
||||||
|
"The video is larger than the download storage you have left — free up space, then try again."
|
||||||
|
)
|
||||||
|
CLIP_TOO_BIG_MESSAGE = (
|
||||||
|
"The clip is larger than the download storage you have left — "
|
||||||
|
"free up space or cut a shorter clip, then try again."
|
||||||
|
)
|
||||||
|
CONVERTED_TOO_BIG_MESSAGE = (
|
||||||
|
"The converted file is larger than the download storage you have left — "
|
||||||
|
"free up space, then try again."
|
||||||
|
)
|
||||||
|
UPLOAD_MISSING_MESSAGE = "The uploaded file is missing."
|
||||||
NETWORK_REDACTED_MESSAGE = "The server could not reach that address."
|
NETWORK_REDACTED_MESSAGE = "The server could not reach that address."
|
||||||
|
|
||||||
# (marker, category) — first match wins, so order most-specific → most-general. Matching is
|
# (marker, category) — first match wins, so order most-specific → most-general. Matching is
|
||||||
@@ -61,6 +76,7 @@ _RULES: tuple[tuple[str, str], ...] = (
|
|||||||
# Our own refusal: the worker found the user already at their storage cap (SEC-13). Placed
|
# Our own refusal: the worker found the user already at their storage cap (SEC-13). Placed
|
||||||
# first — it is our wording, so it can never be a false match on an upstream message.
|
# first — it is our wording, so it can never be a false match on an upstream message.
|
||||||
("download storage is full", "quota_exceeded"),
|
("download storage is full", "quota_exceeded"),
|
||||||
|
("larger than the download storage you have left", "quota_exceeded"),
|
||||||
# Our own redaction of a transport failure (see NETWORK_REDACTED_MESSAGE) — our wording too.
|
# Our own redaction of a transport failure (see NETWORK_REDACTED_MESSAGE) — our wording too.
|
||||||
(NETWORK_REDACTED_MESSAGE.lower(), "network"),
|
(NETWORK_REDACTED_MESSAGE.lower(), "network"),
|
||||||
# Our own verdicts on a post link (adapters.py) — our wording too: its page holds no single video,
|
# Our own verdicts on a post link (adapters.py) — our wording too: its page holds no single video,
|
||||||
@@ -98,6 +114,8 @@ _RULES: tuple[tuple[str, str], ...] = (
|
|||||||
# An edit's source download went away — our own wording, and MORE specific than the generic
|
# An edit's source download went away — our own wording, and MORE specific than the generic
|
||||||
# "no longer available" below, so it must be matched first.
|
# "no longer available" below, so it must be matched first.
|
||||||
("source download is no longer available", "source_missing"),
|
("source download is no longer available", "source_missing"),
|
||||||
|
# An upload's original went away before its conversion ran — our wording (UPLOAD_MISSING_MESSAGE).
|
||||||
|
(UPLOAD_MISSING_MESSAGE.lower().rstrip("."), "upload_missing"),
|
||||||
("source file is missing", "source_missing"),
|
("source file is missing", "source_missing"),
|
||||||
# Private / deleted / removed / terminated.
|
# Private / deleted / removed / terminated.
|
||||||
("private video", "unavailable"),
|
("private video", "unavailable"),
|
||||||
@@ -126,6 +144,13 @@ _RULES: tuple[tuple[str, str], ...] = (
|
|||||||
("postprocessing", "postprocess"),
|
("postprocessing", "postprocess"),
|
||||||
("produced no output file", "postprocess"),
|
("produced no output file", "postprocess"),
|
||||||
("no output file", "postprocess"),
|
("no output file", "postprocess"),
|
||||||
|
# yt-dlp's own verdict that its extractor did not understand the site's answer: it appends this
|
||||||
|
# request for a bug report to an UNEXPECTED extraction failure, never to an expected one (a private,
|
||||||
|
# removed or geo-blocked video says so in words matched above). Measured 2026-10-12: a TikTok link
|
||||||
|
# failed with it on every attempt under yt-dlp 2026.07.04 and downloads under 2026.08.19 — the
|
||||||
|
# site changed and a newer yt-dlp is the cure. After the media-format and post-processing rules,
|
||||||
|
# whose texts can carry the same request; before the network ones, which are about the transport.
|
||||||
|
("please report this issue", "site_changed"),
|
||||||
# Network / transport.
|
# Network / transport.
|
||||||
("unable to download", "network"),
|
("unable to download", "network"),
|
||||||
("http error", "network"),
|
("http error", "network"),
|
||||||
@@ -263,6 +288,15 @@ def classify(msg: str | None) -> str:
|
|||||||
return "unknown"
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def classify_conversion(msg: str | None) -> str:
|
||||||
|
"""The category of an upload's failed conversion. The generic ffmpeg categories speak of a
|
||||||
|
download ("the video downloaded but…"), which an upload never was: a failed encode or an
|
||||||
|
unrecognised failure of one is `convert_failed`; a category of its own (a full quota, a missing
|
||||||
|
original) stays as it is."""
|
||||||
|
code = classify(msg)
|
||||||
|
return "convert_failed" if code in ("postprocess", "unknown") else code
|
||||||
|
|
||||||
|
|
||||||
def diagnose(
|
def diagnose(
|
||||||
msg: str | None, *, url: str | None = None, cookie_attached: bool = False
|
msg: str | None, *, url: str | None = None, cookie_attached: bool = False
|
||||||
) -> Diagnosis:
|
) -> Diagnosis:
|
||||||
|
|||||||
@@ -0,0 +1,312 @@
|
|||||||
|
"""The folders of the Files library — where each of a user's files sits.
|
||||||
|
|
||||||
|
A folder is a row, never a directory: on disk a file stays where the download layout put it (one
|
||||||
|
shared asset can be held by several users), so a rename or a move is a column update. Three kinds:
|
||||||
|
|
||||||
|
- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips,
|
||||||
|
uploads, one per other site). Its `source_key` names the origin; the client renders its name
|
||||||
|
from the key, in the reader's language.
|
||||||
|
- an OWN folder is the user's, with the name they gave it.
|
||||||
|
- the HUB folder (one per user, made when they first give a machine a token for it, `hub_folder`) is
|
||||||
|
where they swap files with that machine. Its name, like a source folder's, is the client's; a file
|
||||||
|
may be moved in like into an own folder, and it keeps a file until it is deleted.
|
||||||
|
|
||||||
|
Every file of a user is in exactly one folder. The folder a new file lands in is decided here
|
||||||
|
(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file
|
||||||
|
moves only into one of the user's own folders, their Hub folder, or back to its own source folder
|
||||||
|
(`move`): a YouTube file in the TikTok folder would be a lie the folder's name tells.
|
||||||
|
|
||||||
|
Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
|
||||||
|
folder was made; the user can change it), Uploads, own folders and the Hub keep it until it is deleted. A
|
||||||
|
done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user
|
||||||
|
kept it — and its `ready_at`, the moment it finished (`stamp_done`).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from enum import StrEnum
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import sysconfig
|
||||||
|
from app.downloads.status import JobStatus
|
||||||
|
from app.models import DownloadJob, FileFolder, MediaAsset
|
||||||
|
|
||||||
|
|
||||||
|
class FolderKind(StrEnum):
|
||||||
|
SOURCE = "source"
|
||||||
|
OWN = "own"
|
||||||
|
HUB = "hub"
|
||||||
|
|
||||||
|
|
||||||
|
class SourceKey(StrEnum):
|
||||||
|
"""The fixed origins. Any other site is `site:<host>` (see `site_key`)."""
|
||||||
|
|
||||||
|
YOUTUBE = "youtube"
|
||||||
|
PLEX = "plex"
|
||||||
|
UPLOAD = "upload"
|
||||||
|
|
||||||
|
|
||||||
|
SITE_PREFIX = "site:"
|
||||||
|
|
||||||
|
# A host's leading label that only says "the same site, another front door".
|
||||||
|
_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
|
||||||
|
_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
|
||||||
|
|
||||||
|
|
||||||
|
def site_key(url: str) -> str:
|
||||||
|
"""The source key of a page on some site: `site:<host>` without its front door (www., m.),
|
||||||
|
or the YouTube key for a YouTube address that arrived as a plain URL."""
|
||||||
|
host = (urlsplit(url).hostname or "").lower().rstrip(".")
|
||||||
|
for door in _FRONT_DOORS:
|
||||||
|
if host.startswith(door) and host.count(".") > 1:
|
||||||
|
host = host[len(door) :]
|
||||||
|
break
|
||||||
|
if host in _YOUTUBE_HOSTS:
|
||||||
|
return SourceKey.YOUTUBE
|
||||||
|
return f"{SITE_PREFIX}{host or 'unknown'}"
|
||||||
|
|
||||||
|
|
||||||
|
def source_key_for(db: Session, job: DownloadJob) -> str:
|
||||||
|
"""The origin a job's file came from — the source folder it belongs in.
|
||||||
|
|
||||||
|
The one recorded when it was placed, else: youtube → YouTube; a clip → the clip's source
|
||||||
|
(YouTube or Plex); an upload → Uploads; a URL → its site; an edit → wherever its parent's file
|
||||||
|
came from (its parent job, or, once that is deleted, the parent's asset)."""
|
||||||
|
if job.source_key:
|
||||||
|
return job.source_key
|
||||||
|
kind = job.source_kind
|
||||||
|
if kind == "youtube":
|
||||||
|
return SourceKey.YOUTUBE
|
||||||
|
if kind == "upload":
|
||||||
|
return SourceKey.UPLOAD
|
||||||
|
if kind == "clip":
|
||||||
|
return (
|
||||||
|
SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE
|
||||||
|
)
|
||||||
|
if kind == "url":
|
||||||
|
return site_key(job.source_ref)
|
||||||
|
if kind == "edit":
|
||||||
|
parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None
|
||||||
|
if parent is not None and parent.id != job.id:
|
||||||
|
return source_key_for(db, parent)
|
||||||
|
asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None
|
||||||
|
if asset is not None:
|
||||||
|
if asset.source_kind == "youtube":
|
||||||
|
return SourceKey.YOUTUBE
|
||||||
|
if asset.source_kind == "upload":
|
||||||
|
return SourceKey.UPLOAD
|
||||||
|
if asset.source_kind == "url":
|
||||||
|
return site_key(asset.source_ref)
|
||||||
|
return SourceKey.UPLOAD
|
||||||
|
|
||||||
|
|
||||||
|
def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
|
||||||
|
"""The user's source folder for `key`, made on first use. A race on the unique (user, key) is
|
||||||
|
settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
|
||||||
|
stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
|
||||||
|
folder = db.execute(stmt).scalar_one_or_none()
|
||||||
|
if folder is not None:
|
||||||
|
return folder
|
||||||
|
retention = (
|
||||||
|
None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days")
|
||||||
|
)
|
||||||
|
folder = FileFolder(
|
||||||
|
user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with db.begin_nested():
|
||||||
|
db.add(folder)
|
||||||
|
db.flush()
|
||||||
|
# silent-ok: the expected outcome of the race the savepoint exists for — another request made
|
||||||
|
# the same folder first; re-selecting it is the whole handling.
|
||||||
|
except IntegrityError:
|
||||||
|
folder = db.execute(stmt).scalar_one()
|
||||||
|
return folder
|
||||||
|
|
||||||
|
|
||||||
|
def hub_folder(db: Session, user_id: int) -> FileFolder:
|
||||||
|
"""The user's Hub folder, made on first use; a race on its one-per-user index is settled like
|
||||||
|
`source_folder`'s."""
|
||||||
|
stmt = select(FileFolder).where(
|
||||||
|
FileFolder.user_id == user_id, FileFolder.kind == FolderKind.HUB
|
||||||
|
)
|
||||||
|
folder = db.execute(stmt).scalar_one_or_none()
|
||||||
|
if folder is not None:
|
||||||
|
return folder
|
||||||
|
folder = FileFolder(user_id=user_id, kind=FolderKind.HUB)
|
||||||
|
try:
|
||||||
|
with db.begin_nested():
|
||||||
|
db.add(folder)
|
||||||
|
db.flush()
|
||||||
|
# silent-ok: another request made the Hub folder first; re-selecting it is the whole handling.
|
||||||
|
except IntegrityError:
|
||||||
|
folder = db.execute(stmt).scalar_one()
|
||||||
|
return folder
|
||||||
|
|
||||||
|
|
||||||
|
def place(db: Session, job: DownloadJob) -> None:
|
||||||
|
"""Put a new job's file in its source folder, and record that origin on it. Called wherever a
|
||||||
|
job is created; the job must carry its `user_id`, `source_kind` and source refs already."""
|
||||||
|
key = source_key_for(db, job)
|
||||||
|
job.source_key = key
|
||||||
|
job.folder_id = source_folder(db, job.user_id, key).id
|
||||||
|
|
||||||
|
|
||||||
|
def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
|
||||||
|
"""When a file that finishes now goes: its folder's retention from now, or never (kept, or a
|
||||||
|
folder that keeps files)."""
|
||||||
|
if kept or folder_id is None:
|
||||||
|
return None
|
||||||
|
# Read under a share lock, held to the caller's commit: a retention change (`set_retention` locks
|
||||||
|
# the folder) waits for this file and restamps it too, or this read waits for the change and
|
||||||
|
# takes the new value — never the old value stamped on a file the change did not see.
|
||||||
|
days = db.execute(
|
||||||
|
select(FileFolder.retention_days)
|
||||||
|
.where(FileFolder.id == folder_id)
|
||||||
|
.with_for_update(read=True)
|
||||||
|
).scalar()
|
||||||
|
return None if days is None else datetime.now(timezone.utc) + timedelta(days=days)
|
||||||
|
|
||||||
|
|
||||||
|
def stamp_done(db: Session, job: DownloadJob) -> None:
|
||||||
|
"""Stamp a job that just became done (in this session): when it did, and its expiry."""
|
||||||
|
job.ready_at = datetime.now(timezone.utc)
|
||||||
|
job.expires_at = done_expiry(db, job.folder_id, job.kept)
|
||||||
|
job.expiry_warned = False
|
||||||
|
|
||||||
|
|
||||||
|
class FolderError(Exception):
|
||||||
|
"""A folder change the rules refuse; `reason` is the refusal the route answers with."""
|
||||||
|
|
||||||
|
def __init__(self, reason: str) -> None:
|
||||||
|
super().__init__(reason)
|
||||||
|
self.reason = reason
|
||||||
|
|
||||||
|
|
||||||
|
def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
|
||||||
|
"""The user's folder `folder_id` that a file is about to be put in — held (FOR SHARE) until the
|
||||||
|
caller's transaction ends, so a delete of the folder waits for the file and then takes it home
|
||||||
|
with the rest (`delete_own` locks it FOR UPDATE), never failing on its foreign key. `hold=False`
|
||||||
|
only looks (a check ahead of a long wait, which must not hold a lock through it). Another user's
|
||||||
|
folder answers like a missing one."""
|
||||||
|
stmt = select(FileFolder).where(FileFolder.id == folder_id)
|
||||||
|
if hold:
|
||||||
|
stmt = stmt.with_for_update(read=True)
|
||||||
|
folder = db.execute(stmt.execution_options(populate_existing=True)).scalar_one_or_none()
|
||||||
|
if folder is None or folder.user_id != user_id:
|
||||||
|
raise FolderError("file_folder_gone")
|
||||||
|
return folder
|
||||||
|
|
||||||
|
|
||||||
|
def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
|
||||||
|
"""The folder an upload asked to land in, if it may: one of the user's own folders, their Hub
|
||||||
|
folder, or their Uploads folder. A file put in a site's folder would claim an origin it does not
|
||||||
|
have."""
|
||||||
|
folder = own_folder(db, user_id, folder_id, hold=hold)
|
||||||
|
if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD:
|
||||||
|
raise FolderError("file_folder_not_its_source")
|
||||||
|
return folder
|
||||||
|
|
||||||
|
|
||||||
|
def restamp(db: Session, job: DownloadJob) -> None:
|
||||||
|
"""Start a file's clock again from now: it moved to another folder, or its user stopped keeping
|
||||||
|
it. A file not done yet gets its expiry when it finishes (`stamp_done`), so it has none now."""
|
||||||
|
done = job.status == JobStatus.DONE
|
||||||
|
job.expires_at = done_expiry(db, job.folder_id, job.kept) if done else None
|
||||||
|
job.expiry_warned = False
|
||||||
|
|
||||||
|
|
||||||
|
def set_kept(db: Session, job: DownloadJob, kept: bool) -> None:
|
||||||
|
"""Keep a file until its user deletes it, or hand it back to its folder's retention."""
|
||||||
|
job.kept = kept
|
||||||
|
restamp(db, job)
|
||||||
|
|
||||||
|
|
||||||
|
def move(db: Session, job: DownloadJob, folder: FileFolder) -> None:
|
||||||
|
"""Put a file in `folder` — one of its user's own folders, their Hub folder, or the source folder
|
||||||
|
it came from.
|
||||||
|
The caller has checked that `folder` is the job's user's."""
|
||||||
|
if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job):
|
||||||
|
raise FolderError("file_folder_not_its_source")
|
||||||
|
if folder.id == job.folder_id:
|
||||||
|
return
|
||||||
|
job.folder_id = folder.id
|
||||||
|
restamp(db, job)
|
||||||
|
|
||||||
|
|
||||||
|
def _hold_for_change(db: Session, folder: FileFolder) -> None:
|
||||||
|
"""Lock `folder` FOR UPDATE and read it afresh into the same object: a change waits out a file
|
||||||
|
being put in it or stamped from it right now (`own_folder`, `done_expiry`), and works from what
|
||||||
|
another change that landed first left, not from what the caller read before the wait. The row is
|
||||||
|
consumed on purpose: an ORM result refreshes the object only as its rows are fetched."""
|
||||||
|
fresh = db.execute(
|
||||||
|
select(FileFolder)
|
||||||
|
.where(FileFolder.id == folder.id)
|
||||||
|
.with_for_update()
|
||||||
|
.execution_options(populate_existing=True)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if fresh is None:
|
||||||
|
raise FolderError("file_folder_gone")
|
||||||
|
|
||||||
|
|
||||||
|
def set_retention(db: Session, folder: FileFolder, days: int | None) -> int | None:
|
||||||
|
"""Change how long `folder` keeps a file (None = until deleted). The done files already in it
|
||||||
|
move by the difference — each keeps the age it has — but none ends up closer than the GC's
|
||||||
|
warning window, so a shortened retention still warns before it takes anything. Returns the
|
||||||
|
retention it replaced (equal to `days` when nothing changed)."""
|
||||||
|
_hold_for_change(db, folder) # so the file arriving now is among the ones restamped below
|
||||||
|
old = folder.retention_days
|
||||||
|
if old == days:
|
||||||
|
return old
|
||||||
|
folder.retention_days = days
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
floor = now + timedelta(days=sysconfig.get_int(db, "download_gc_grace_days"))
|
||||||
|
# Locked as they are read: one moved out or deleted meanwhile is no longer this folder's to time.
|
||||||
|
jobs = db.execute(
|
||||||
|
select(DownloadJob)
|
||||||
|
.where(
|
||||||
|
DownloadJob.folder_id == folder.id,
|
||||||
|
DownloadJob.status == JobStatus.DONE,
|
||||||
|
DownloadJob.kept.is_(False),
|
||||||
|
)
|
||||||
|
.with_for_update()
|
||||||
|
.execution_options(populate_existing=True)
|
||||||
|
).scalars()
|
||||||
|
for job in jobs:
|
||||||
|
if days is None:
|
||||||
|
job.expires_at = None
|
||||||
|
elif old is None or job.expires_at is None:
|
||||||
|
job.expires_at = max(now + timedelta(days=days), floor)
|
||||||
|
else:
|
||||||
|
job.expires_at = max(job.expires_at + timedelta(days=days - old), floor)
|
||||||
|
job.expiry_warned = False
|
||||||
|
return old
|
||||||
|
|
||||||
|
|
||||||
|
def delete_own(db: Session, folder: FileFolder) -> int:
|
||||||
|
"""Delete one of a user's own folders. Its files are not deleted: each goes back to the source
|
||||||
|
folder it came from, its clock starting there now. Returns how many moved."""
|
||||||
|
if folder.kind != FolderKind.OWN:
|
||||||
|
raise FolderError("file_folder_not_own")
|
||||||
|
_hold_for_change(db, folder) # so the file arriving now is taken home too
|
||||||
|
# Locked as they are read: a file moved out meanwhile stays where it went, and one deleted
|
||||||
|
# meanwhile is not written back (its gone row would fail the delete).
|
||||||
|
jobs = (
|
||||||
|
db.execute(
|
||||||
|
select(DownloadJob)
|
||||||
|
.where(DownloadJob.folder_id == folder.id)
|
||||||
|
.with_for_update()
|
||||||
|
.execution_options(populate_existing=True)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for job in jobs:
|
||||||
|
job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
|
||||||
|
restamp(db, job)
|
||||||
|
db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete
|
||||||
|
db.delete(folder)
|
||||||
|
return len(jobs)
|
||||||
@@ -105,6 +105,28 @@ def log_safe(text: object) -> str:
|
|||||||
return _PATH_RE.sub("…", text)
|
return _PATH_RE.sub("…", text)
|
||||||
|
|
||||||
|
|
||||||
|
# yt-dlp's HTTP downloader, refusing a file announced larger than `max_filesize`, logs this line and
|
||||||
|
# RETURNS — no exception and no file (measured; test_formats holds it against the pinned yt-dlp).
|
||||||
|
_OVER_CAP_LINE = "File is larger than max-filesize"
|
||||||
|
_OVER_CAP: contextvars.ContextVar[list[bool] | None] = contextvars.ContextVar(
|
||||||
|
"formats_over_cap", default=None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def watching_size_cap() -> Iterator[list[bool]]:
|
||||||
|
"""While yt-dlp works inside it: a one-slot flag, set when it refused a file over the size cap
|
||||||
|
(`build_ydl_opts(max_bytes=…)`). It then returns as if it had downloaded, and all the caller
|
||||||
|
would find is no file — a processing fault, to anyone reading it, for what is the owner's
|
||||||
|
storage. A context variable, like `hiding`: yt-dlp's logger calls run on the extraction's thread."""
|
||||||
|
flag = [False]
|
||||||
|
token = _OVER_CAP.set(flag)
|
||||||
|
try:
|
||||||
|
yield flag
|
||||||
|
finally:
|
||||||
|
_OVER_CAP.reset(token)
|
||||||
|
|
||||||
|
|
||||||
class _YdlLog:
|
class _YdlLog:
|
||||||
"""yt-dlp's logger protocol, pointed at ours.
|
"""yt-dlp's logger protocol, pointed at ours.
|
||||||
|
|
||||||
@@ -124,6 +146,8 @@ class _YdlLog:
|
|||||||
is the one line whose level should say so."""
|
is the one line whose level should say so."""
|
||||||
|
|
||||||
def _emit(self, level: int, msg: str) -> None:
|
def _emit(self, level: int, msg: str) -> None:
|
||||||
|
if _OVER_CAP_LINE in msg and (over := _OVER_CAP.get()) is not None:
|
||||||
|
over[0] = True
|
||||||
# The line goes as an ARGUMENT, not as the format string. yt-dlp writes progress as
|
# The line goes as an ARGUMENT, not as the format string. yt-dlp writes progress as
|
||||||
# `[download] 53.2% of 12.00MiB`, and while stdlib logging only applies `%` formatting to
|
# `[download] 53.2% of 12.00MiB`, and while stdlib logging only applies `%` formatting to
|
||||||
# a record that has args, this way the question never arises — and it is the shape ruff's
|
# a record that has args, this way the question never arises — and it is the shape ruff's
|
||||||
@@ -182,7 +206,9 @@ _DEFAULTS = {
|
|||||||
# v4: EVERY download is clamped to the mp4/H.264/AAC compat profile (no more mkv/webm/VP9/AV1/Opus
|
# v4: EVERY download is clamped to the mp4/H.264/AAC compat profile (no more mkv/webm/VP9/AV1/Opus
|
||||||
# opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped
|
# opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped
|
||||||
# the v3 net — must re-derive through the current browser-safe guarantee.
|
# the v3 net — must re-derive through the current browser-safe guarantee.
|
||||||
_SIG_VERSION = 4
|
# v5: the phone-safe rule (`playable`) also judges the pixel format, so a cached 10-bit / 4:4:4 H.264
|
||||||
|
# asset (which v4 let through as "H.264") re-derives as 8-bit 4:2:0 instead of being reused.
|
||||||
|
_SIG_VERSION = 5
|
||||||
# Audio-only downloads version on their own, so a change to the audio bytes doesn't re-download every
|
# Audio-only downloads version on their own, so a change to the audio bytes doesn't re-download every
|
||||||
# cached video. v5: an extracted .mp3/.m4a is stored as itself instead of remuxed into an .mp4 that
|
# cached video. v5: an extracted .mp3/.m4a is stored as itself instead of remuxed into an .mp4 that
|
||||||
# iPad's WebKit refuses to play, so a cached remuxed audio asset must re-derive.
|
# iPad's WebKit refuses to play, so a cached remuxed audio asset must re-derive.
|
||||||
@@ -219,12 +245,13 @@ def normalize(spec: dict) -> dict:
|
|||||||
s["embed_subs"] = False
|
s["embed_subs"] = False
|
||||||
# Browser-playability guarantee: the Download module exists to download → share → PLAY
|
# Browser-playability guarantee: the Download module exists to download → share → PLAY
|
||||||
# in-browser, so every output must play in EVERY browser, iOS/Safari/WebKit included. The only
|
# in-browser, so every output must play in EVERY browser, iOS/Safari/WebKit included. The only
|
||||||
# universally-safe combo is mp4 + H.264 + AAC/MP3, so we CLAMP the spec to it HERE — the single
|
# universally-safe combo is mp4 + H.264 + AAC (an .mp3/.m4a for audio), so we CLAMP the spec to
|
||||||
# point every path (picker, stored profile, raw API) flows through — instead of only hiding
|
# it HERE — the single point every path (picker, stored profile, raw API) flows through — instead
|
||||||
# options in the UI. Non-mp4 containers (mkv doesn't play at all; webm/VP9/AV1 fail iOS) and any
|
# of only hiding options in the UI. Non-mp4 containers (mkv plays nowhere on iOS; AV1 fails there,
|
||||||
# explicit vcodec are coerced to the compat profile; Opus audio (Safari-flaky as a bare file)
|
# and webm/VP9 plays only on recent devices) and any explicit vcodec are coerced to the compat
|
||||||
# falls back to AAC/m4a. A source that exposes no H.264 is still caught by the worker's
|
# profile; Opus audio falls back to AAC/m4a. A source that exposes no H.264 (or a 10-bit one) is
|
||||||
# browser-safe re-encode (`_ensure_browser_playable`), so playback is guaranteed end-to-end.
|
# still caught by the worker's phone-safe step (`downloads.playable`), so playback is guaranteed
|
||||||
|
# end-to-end.
|
||||||
s["container"] = "mp4"
|
s["container"] = "mp4"
|
||||||
s["vcodec"] = None
|
s["vcodec"] = None
|
||||||
if s["audio_format"] not in ("m4a", "mp3"):
|
if s["audio_format"] not in ("m4a", "mp3"):
|
||||||
@@ -278,8 +305,11 @@ def build_ydl_opts(
|
|||||||
after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra`
|
after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra`
|
||||||
merge so a source adapter can never clobber it.
|
merge so a source adapter can never clobber it.
|
||||||
|
|
||||||
`max_bytes` (SEC-13) is the caller's remaining storage budget; yt-dlp refuses a format larger
|
`max_bytes` (SEC-13) is the caller's remaining storage budget; yt-dlp's HTTP downloader refuses
|
||||||
than it rather than filling the disk and only then being caught by the quota accounting."""
|
a file announced larger than it rather than filling the disk and only then being caught by the
|
||||||
|
quota accounting. Its ffmpeg downloader (a `download_ranges` fetch) does not read it, nor does
|
||||||
|
a fragmented download (YouTube's formats: measured, 718 MB fetched with 1 MB left) — so the
|
||||||
|
pipeline also holds every produced file to what is left by its real size."""
|
||||||
s = normalize(spec)
|
s = normalize(spec)
|
||||||
h = s["max_height"]
|
h = s["max_height"]
|
||||||
opts: dict = {
|
opts: dict = {
|
||||||
@@ -351,7 +381,7 @@ def build_ydl_opts(
|
|||||||
# inside mp4 — a VP9/AV1-in-mp4 file shows a broken player there — so H.264 is selected at the
|
# inside mp4 — a VP9/AV1-in-mp4 file shows a broken player there — so H.264 is selected at the
|
||||||
# SELECTION level, not merely preferred: a sort key can't conjure an H.264 stream a source
|
# SELECTION level, not merely preferred: a sort key can't conjure an H.264 stream a source
|
||||||
# doesn't expose as video-only. A source that ships only non-H.264 is caught after download by
|
# doesn't expose as video-only. A source that ships only non-H.264 is caught after download by
|
||||||
# the worker's browser-safe re-encode (`_ensure_browser_playable`).
|
# the worker's phone-safe step (`downloads.playable`).
|
||||||
if s["mode"] == "v":
|
if s["mode"] == "v":
|
||||||
# Video-only: keep it audio-free. Prefer an H.264 video-only stream, then any video-only,
|
# Video-only: keep it audio-free. Prefer an H.264 video-only stream, then any video-only,
|
||||||
# and only fall to a progressive (audio-bearing) `best` as a last resort.
|
# and only fall to a progressive (audio-bearing) `best` as a last resort.
|
||||||
|
|||||||
+232
-96
@@ -1,82 +1,167 @@
|
|||||||
"""Download retention garbage collector (a scheduler job).
|
"""Download retention garbage collector (a scheduler job).
|
||||||
|
|
||||||
Five passes each run:
|
Six passes each run:
|
||||||
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
|
1. Pre-expiry warning — a done file whose `expires_at` (its folder's retention, `folders`) falls
|
||||||
once (gc_notified flag) so a download isn't silently deleted.
|
within the grace window, and that its user did not keep, warns its user once (`expiry_warned`).
|
||||||
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
2. Expiry — such a file past its `expires_at` leaves the library: the job is deleted and its hold
|
||||||
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
released, so the shared file goes with its last holder.
|
||||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed ready
|
||||||
ready assets until back under it.
|
assets (never an upload, never one a user kept) until back under it; each holder's file leaves
|
||||||
|
their library, with a notice.
|
||||||
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
|
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
|
||||||
passes 1-3 skip them; benign row-count hygiene).
|
passes 1-3 skip them; benign row-count hygiene).
|
||||||
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
|
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
|
||||||
re-download renames an asset, e.g. old accented channel dirs), skipping system trees and
|
re-download renames an asset, e.g. old accented channel dirs, or an upload's stranded file).
|
||||||
anything newer than an hour so an in-flight download is never touched.
|
It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
|
||||||
|
so an in-flight download is never touched.
|
||||||
|
6. Token expiry warning — a machine's token for the library about to run out warns its user once
|
||||||
|
(`apitokens.warn_expiring`): the library's other clock.
|
||||||
|
|
||||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
Runs in the API process (which mounts FILES_ROOT). Pure disk/DB work — no YouTube quota.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, or_, select, update
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs, sysconfig
|
from app import apitokens, logs, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import storage
|
from app.downloads import quota, service, storage
|
||||||
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.models import DownloadJob, MediaAsset
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
# The one dot-prefixed top-level tree the orphan sweep enters. The others are not part of the asset
|
||||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
# layout (user clips, Plex HLS/img caches, in-progress staging, editor storyboards) and are skipped.
|
||||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
_UPLOAD_TREE = ".uploads"
|
||||||
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
|
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
|
||||||
|
|
||||||
|
|
||||||
def _holders(db: Session, asset_id: int) -> list[int]:
|
def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str) -> None:
|
||||||
return list(
|
title = job.display_name or (asset.title if asset else None) or "download"
|
||||||
db.execute(
|
create_notification(
|
||||||
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
|
db,
|
||||||
).scalars()
|
user_id=job.user_id,
|
||||||
|
type=ntype,
|
||||||
|
title=title, # English fallback; the client renders a translated message from data
|
||||||
|
data={
|
||||||
|
"kind": ntype,
|
||||||
|
"title": title,
|
||||||
|
"video_id": asset.source_ref if asset else None,
|
||||||
|
"job_id": job.id,
|
||||||
|
},
|
||||||
|
commit=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
|
def _relock_expired(
|
||||||
title = asset.title or "download"
|
db: Session, job_id: int, asset_id: int | None, now: datetime
|
||||||
for uid in _holders(db, asset.id):
|
) -> DownloadJob | None:
|
||||||
create_notification(
|
"""Re-read an expired job under the lock order the user routes take (`own_job_for_update`: the
|
||||||
db,
|
asset, then the job) and say whether it is still to go. Between the pass's SELECT and here a
|
||||||
user_id=uid,
|
user may have deleted or re-downloaded it (dropping it again would release its asset a second
|
||||||
type=ntype,
|
time, freeing a file another job still holds) or kept or moved it (it is no longer expired)."""
|
||||||
title=title, # English fallback; the client renders a translated message from data
|
if asset_id is not None:
|
||||||
data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
|
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||||
commit=False,
|
job = db.execute(
|
||||||
|
select(DownloadJob)
|
||||||
|
.where(DownloadJob.id == job_id)
|
||||||
|
.with_for_update()
|
||||||
|
.execution_options(populate_existing=True)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
still = (
|
||||||
|
job is not None
|
||||||
|
and job.asset_id == asset_id
|
||||||
|
and job.status == JobStatus.DONE
|
||||||
|
and not job.kept
|
||||||
|
and job.expires_at is not None
|
||||||
|
and job.expires_at <= now
|
||||||
|
)
|
||||||
|
return job if still else None
|
||||||
|
|
||||||
|
|
||||||
|
def _mark_warned(db: Session, job_id: int, expires_at: datetime) -> DownloadJob | None:
|
||||||
|
"""Mark one file warned if it still goes when the pass saw it go — not deleted, kept, moved or
|
||||||
|
re-timed since (each of those sets a fresh expiry, and its own warning to come) — and return it.
|
||||||
|
The UPDATE's row lock holds it so until the commit."""
|
||||||
|
hit = db.execute(
|
||||||
|
update(DownloadJob)
|
||||||
|
.where(
|
||||||
|
DownloadJob.id == job_id,
|
||||||
|
DownloadJob.status == JobStatus.DONE,
|
||||||
|
DownloadJob.kept.is_(False),
|
||||||
|
DownloadJob.expiry_warned.is_(False),
|
||||||
|
DownloadJob.expires_at == expires_at,
|
||||||
)
|
)
|
||||||
|
.values(expiry_warned=True)
|
||||||
|
.returning(DownloadJob.id)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
return db.get(DownloadJob, job_id) if hit is not None else None
|
||||||
|
|
||||||
|
|
||||||
def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
def _on_disk(db: Session) -> int:
|
||||||
if asset.rel_path:
|
"""The bytes the cache holds now (every asset whose file is on disk)."""
|
||||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
return (
|
||||||
_notify_holders(db, asset, ntype)
|
db.execute(
|
||||||
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
|
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||||
|
).scalar()
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
def _relock_evictable(db: Session, asset_id: int) -> tuple[MediaAsset | None, list[DownloadJob]]:
|
||||||
"""Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
|
"""Re-read an eviction candidate and its holders under their locks (the asset, then its jobs —
|
||||||
|
the order the user routes take): between the pass's SELECT and here a holder may have kept it,
|
||||||
|
or every holder deleted it."""
|
||||||
|
asset = db.execute(
|
||||||
|
select(MediaAsset)
|
||||||
|
.where(MediaAsset.id == asset_id)
|
||||||
|
.with_for_update()
|
||||||
|
.execution_options(populate_existing=True)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
holders = (
|
||||||
|
db.execute(
|
||||||
|
select(DownloadJob)
|
||||||
|
.where(DownloadJob.asset_id == asset_id)
|
||||||
|
.with_for_update()
|
||||||
|
.execution_options(populate_existing=True)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return asset, list(holders)
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
|
||||||
|
"""Take a file out of its user's library: tell them, release the job's hold on the asset (the
|
||||||
|
file goes with the last holder), delete the row (its shares and links cascade)."""
|
||||||
|
_notify(db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, ntype)
|
||||||
|
service.release_asset(db, job)
|
||||||
|
db.delete(job)
|
||||||
|
|
||||||
|
|
||||||
|
def _sweep_orphan_files(db: Session, files_root: str) -> int:
|
||||||
|
"""Delete media/sidecar files (and now-empty dirs) under FILES_ROOT that no `ready` asset
|
||||||
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
|
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
|
||||||
channel dir, or a poster from a superseded naming scheme). Skips the dot-prefixed system trees
|
channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
|
||||||
and only removes files older than `_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download
|
row names at all. Skips the other dot-prefixed system trees and only removes files older than
|
||||||
is never deleted, and only files NOT owned by a surviving asset."""
|
`_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
|
||||||
root = Path(download_root)
|
NOT owned by a surviving asset."""
|
||||||
|
root = Path(files_root)
|
||||||
if not root.is_dir():
|
if not root.is_dir():
|
||||||
return 0
|
return 0
|
||||||
keep: set[Path] = set()
|
keep: set[Path] = set()
|
||||||
|
# An upload's file is on disk from ingest on, whatever its state (a queued conversion's original,
|
||||||
|
# a failed one kept for Retry), so every upload row keeps its file — not only a ready one.
|
||||||
for rel in db.execute(
|
for rel in db.execute(
|
||||||
select(MediaAsset.rel_path).where(
|
select(MediaAsset.rel_path).where(
|
||||||
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
or_(MediaAsset.status == "ready", MediaAsset.source_kind == "upload"),
|
||||||
|
MediaAsset.rel_path.is_not(None),
|
||||||
)
|
)
|
||||||
).scalars():
|
).scalars():
|
||||||
media = (root / rel).resolve()
|
media = (root / rel).resolve()
|
||||||
@@ -99,7 +184,10 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
|||||||
|
|
||||||
unreclaimed = 0
|
unreclaimed = 0
|
||||||
for entry in root.iterdir():
|
for entry in root.iterdir():
|
||||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
# .edits / .plex-* / .staging / .storyboards + any dotfile — but `.uploads` is swept: a
|
||||||
|
# conversion stopped between placing the new file and the commit that points at it (or the
|
||||||
|
# unlink of the original after it) leaves a file no row names, and nothing else finds it.
|
||||||
|
if entry.name.startswith(".") and entry.name != _UPLOAD_TREE:
|
||||||
continue
|
continue
|
||||||
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
||||||
for f in files:
|
for f in files:
|
||||||
@@ -143,74 +231,95 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
||||||
warned = expired = evicted = reaped = 0
|
warned = expired = evicted = reaped = 0
|
||||||
|
|
||||||
# 1. Pre-expiry warnings.
|
# 1. Pre-expiry warnings — one per file, the first run inside the grace window. Each mark is
|
||||||
|
# conditional on the expiry the SELECT saw and committed on its own: a file deleted since is
|
||||||
|
# passed over (a write to its gone row would fail the whole run), and one kept, moved or re-timed
|
||||||
|
# since keeps its fresh expiry unwarned.
|
||||||
warn_before = now + timedelta(days=grace_days)
|
warn_before = now + timedelta(days=grace_days)
|
||||||
soon = (
|
soon = db.execute(
|
||||||
db.execute(
|
select(DownloadJob.id, DownloadJob.expires_at).where(
|
||||||
select(MediaAsset).where(
|
DownloadJob.status == JobStatus.DONE,
|
||||||
MediaAsset.status == "ready",
|
DownloadJob.kept.is_(False),
|
||||||
MediaAsset.gc_notified.is_(False),
|
DownloadJob.expiry_warned.is_(False),
|
||||||
MediaAsset.expires_at.is_not(None),
|
DownloadJob.expires_at.is_not(None),
|
||||||
MediaAsset.expires_at > now,
|
DownloadJob.expires_at > now,
|
||||||
MediaAsset.expires_at <= warn_before,
|
DownloadJob.expires_at <= warn_before,
|
||||||
)
|
|
||||||
)
|
)
|
||||||
.scalars()
|
).all()
|
||||||
.all()
|
db.rollback() # the pass-1 snapshot holds no locks; each mark takes its own below
|
||||||
)
|
for job_id, expires_at in soon:
|
||||||
for asset in soon:
|
job = _mark_warned(db, job_id, expires_at)
|
||||||
_notify_holders(db, asset, "download_expiring")
|
if job is not None:
|
||||||
asset.gc_notified = True
|
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||||
warned += 1
|
_notify(db, job, asset, "download_expiring")
|
||||||
if soon:
|
warned += 1
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
# 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
|
||||||
dead = (
|
# Each file is re-checked under its locks and committed on its own: the release unlinks the
|
||||||
db.execute(
|
# file at once, so it must never act on what the SELECT saw a while ago.
|
||||||
select(MediaAsset).where(
|
dead = db.execute(
|
||||||
MediaAsset.status.in_(("ready", "error")),
|
select(DownloadJob.id, DownloadJob.asset_id).where(
|
||||||
MediaAsset.expires_at.is_not(None),
|
DownloadJob.status == JobStatus.DONE,
|
||||||
MediaAsset.expires_at <= now,
|
DownloadJob.kept.is_(False),
|
||||||
)
|
DownloadJob.expires_at.is_not(None),
|
||||||
|
DownloadJob.expires_at <= now,
|
||||||
)
|
)
|
||||||
.scalars()
|
).all()
|
||||||
.all()
|
db.rollback() # the pass-2 snapshot holds no locks; each file takes its own below
|
||||||
)
|
for job_id, asset_id in dead:
|
||||||
for asset in dead:
|
job = _relock_expired(db, job_id, asset_id, now)
|
||||||
_delete_asset(db, asset, "download_expired")
|
if job is not None:
|
||||||
expired += 1
|
_drop_job(db, job, "download_expired")
|
||||||
if dead:
|
expired += 1
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
# 3. LRU eviction over the total cache cap.
|
# 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to
|
||||||
|
# fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
|
||||||
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
||||||
if cap > 0:
|
if cap > 0:
|
||||||
total = (
|
if _on_disk(db) > cap:
|
||||||
db.execute(
|
kept_assets = select(DownloadJob.asset_id).where(
|
||||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
|
||||||
MediaAsset.status == "ready"
|
)
|
||||||
)
|
candidate_ids = (
|
||||||
).scalar()
|
|
||||||
or 0
|
|
||||||
)
|
|
||||||
if total > cap:
|
|
||||||
candidates = (
|
|
||||||
db.execute(
|
db.execute(
|
||||||
select(MediaAsset)
|
select(MediaAsset.id)
|
||||||
.where(MediaAsset.status == "ready")
|
.where(
|
||||||
|
MediaAsset.status == "ready",
|
||||||
|
MediaAsset.source_kind != "upload",
|
||||||
|
MediaAsset.id.not_in(kept_assets),
|
||||||
|
)
|
||||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
for asset in candidates:
|
db.rollback()
|
||||||
if total <= cap:
|
for asset_id in candidate_ids:
|
||||||
|
# Counted afresh before each: a file deleted since (this one, an upload, a candidate
|
||||||
|
# further down) has freed its space, and no other user's file goes for that space.
|
||||||
|
if _on_disk(db) <= cap:
|
||||||
break
|
break
|
||||||
total -= asset.size_bytes or 0
|
# Locked and re-read like an expiry: since the SELECT a holder may have kept it,
|
||||||
_delete_asset(db, asset, "download_evicted")
|
# every holder deleted it, or someone asked for it again (a fresh cache hit).
|
||||||
|
asset, holders = _relock_evictable(db, asset_id)
|
||||||
|
if (
|
||||||
|
asset is None
|
||||||
|
or asset.status != "ready"
|
||||||
|
or any(j.kept for j in holders)
|
||||||
|
or (asset.last_access_at is not None and asset.last_access_at > now)
|
||||||
|
):
|
||||||
|
db.commit()
|
||||||
|
continue
|
||||||
|
for job in holders:
|
||||||
|
_notify(db, job, asset, "download_evicted")
|
||||||
|
db.delete(job)
|
||||||
|
if asset.rel_path:
|
||||||
|
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||||
|
db.delete(asset)
|
||||||
evicted += 1
|
evicted += 1
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
||||||
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
||||||
@@ -232,9 +341,35 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
if orphans:
|
if orphans:
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
# 4b. Reap unheld uploads. An upload carries no expires_at, is never evicted and never re-enqueued,
|
||||||
|
# so the passes above never free one that lost its last holder outside `release_asset` — a
|
||||||
|
# conversion that finished after its job was cancelled, or one cancelled mid-run (the asset
|
||||||
|
# was `downloading` then). Its file, ready or still the unconverted original, is at `rel_path`.
|
||||||
|
unheld = (
|
||||||
|
db.execute(
|
||||||
|
select(MediaAsset).where(
|
||||||
|
MediaAsset.source_kind == "upload",
|
||||||
|
MediaAsset.ref_count == 0,
|
||||||
|
MediaAsset.status != "downloading",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for asset in unheld:
|
||||||
|
if asset.rel_path: # no holder is left to tell, so no notice — just the file and the row
|
||||||
|
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||||
|
db.delete(asset)
|
||||||
|
reaped += 1
|
||||||
|
if unheld:
|
||||||
|
db.commit()
|
||||||
|
|
||||||
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
|
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
|
||||||
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
|
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
|
||||||
swept = _sweep_orphan_files(db, settings.download_root)
|
swept = _sweep_orphan_files(db, settings.files_root)
|
||||||
|
|
||||||
|
# 6. A machine's token about to run out.
|
||||||
|
tokens_warned = apitokens.warn_expiring(db)
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"warned": warned,
|
"warned": warned,
|
||||||
@@ -242,6 +377,7 @@ def run_download_gc(db: Session) -> dict:
|
|||||||
"evicted": evicted,
|
"evicted": evicted,
|
||||||
"reaped": reaped,
|
"reaped": reaped,
|
||||||
"swept": swept,
|
"swept": swept,
|
||||||
|
"tokens_warned": tokens_warned,
|
||||||
}
|
}
|
||||||
log.info("download_gc %s", result)
|
log.info("download_gc %s", result)
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -68,8 +68,8 @@ def _reset_asset_pending(asset_id: int) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _staging_dir(asset_id: int) -> Path:
|
def _staging_dir(asset_id: int) -> Path:
|
||||||
return Path(settings.download_root) / ".staging" / f"asset-{asset_id}"
|
return Path(settings.files_root) / ".staging" / f"asset-{asset_id}"
|
||||||
|
|
||||||
|
|
||||||
def _edit_staging_dir(asset_id: int) -> Path:
|
def _edit_staging_dir(asset_id: int) -> Path:
|
||||||
return Path(settings.download_root) / ".staging" / f"edit-{asset_id}"
|
return Path(settings.files_root) / ".staging" / f"edit-{asset_id}"
|
||||||
|
|||||||
@@ -1,10 +1,15 @@
|
|||||||
"""Public "share by link" helpers for the download center.
|
"""Public "share by link" helpers for the download center.
|
||||||
|
|
||||||
A DownloadLink is a capability URL: the unguessable `token` grants anyone read access to one
|
A DownloadLink is a capability URL: the unguessable token grants anyone read access to one
|
||||||
download's file on the login-free `/watch/{token}` page. Password-protected links additionally
|
download's file, or to the files in one of its owner's own folders (`folder_files`), on the
|
||||||
require a short-lived signed **grant** on the file request — because a `<video src>` can't send a
|
login-free `/watch/{token}` page. So the token is never stored as written:
|
||||||
header, the password is exchanged once at `/unlock` for an HMAC grant appended to the file URL,
|
the link is found by its SHA-256, and the owner's copy is Fernet-encrypted — a database copy alone
|
||||||
so the raw password never rides in the media URL (or server logs).
|
opens nothing. Without that copy (no TOKEN_ENCRYPTION_KEY, or a key since changed) the link still
|
||||||
|
opens; only its address cannot be shown again, and the owner makes a new link to share.
|
||||||
|
|
||||||
|
Password-protected links additionally require a short-lived signed **grant** on the file request —
|
||||||
|
because a `<video src>` can't send a header, the password is exchanged once at `/unlock` for an HMAC
|
||||||
|
grant appended to the file URL, so the raw password never rides in the media URL (or server logs).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
@@ -12,16 +17,71 @@ import hmac
|
|||||||
import secrets
|
import secrets
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import security
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.models import DownloadLink
|
from app.downloads import storage
|
||||||
|
from app.downloads.status import JobStatus
|
||||||
|
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||||
|
|
||||||
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
|
_GRANT_TTL = 6 * 3600 # a watch session's grant is good for 6h, then the viewer re-unlocks
|
||||||
|
# The most files a folder link lists: as many as the owner's own Files list shows (`routes.files.
|
||||||
|
# MAX_LIST`; a test holds the two equal — this layer may not import that one).
|
||||||
|
FOLDER_MAX = 500
|
||||||
|
|
||||||
|
|
||||||
def new_token() -> str:
|
def new_token() -> str:
|
||||||
return secrets.token_urlsafe(24) # ~32 chars, 192 bits
|
return secrets.token_urlsafe(24) # ~32 chars, 192 bits
|
||||||
|
|
||||||
|
|
||||||
|
def seal(link: DownloadLink, token: str) -> None:
|
||||||
|
"""Store `token` on `link` the only ways it is kept: its hash, and its encrypted copy."""
|
||||||
|
link.token_hash = security.hash_token(token)
|
||||||
|
link.token_enc = security.encrypt(token) if security.can_encrypt() else None
|
||||||
|
|
||||||
|
|
||||||
|
def find(db: Session, token: str) -> DownloadLink | None:
|
||||||
|
"""The link a `/watch` token names, by its hash."""
|
||||||
|
return db.execute(
|
||||||
|
select(DownloadLink).where(DownloadLink.token_hash == security.hash_token(token))
|
||||||
|
).scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
def recover(link: DownloadLink) -> str | None:
|
||||||
|
"""The link's token from its encrypted copy, or None when there is none this key can read."""
|
||||||
|
return security.decrypt_optional(link.token_enc)
|
||||||
|
|
||||||
|
|
||||||
|
def in_folder(folder: FileFolder) -> tuple:
|
||||||
|
"""Which jobs a link to `folder` reaches: the done files in it NOW, the folder owner's. A file
|
||||||
|
moved out stops opening through the link at its next request; one moved in starts."""
|
||||||
|
return (
|
||||||
|
DownloadJob.folder_id == folder.id,
|
||||||
|
DownloadJob.user_id == folder.user_id,
|
||||||
|
DownloadJob.status == JobStatus.DONE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def folder_files(db: Session, folder: FileFolder) -> list[tuple[DownloadJob, MediaAsset]]:
|
||||||
|
"""What a link to `folder` shows — its page lists exactly these, and its link card counts them:
|
||||||
|
each file `in_folder` whose asset is ready and whose bytes are on disk (so the list never offers
|
||||||
|
a dead row), newest first, at most `FOLDER_MAX`."""
|
||||||
|
rows = db.execute(
|
||||||
|
select(DownloadJob, MediaAsset)
|
||||||
|
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||||
|
.where(*in_folder(folder), MediaAsset.status == "ready")
|
||||||
|
.order_by(DownloadJob.ready_at.desc().nulls_last(), DownloadJob.id.desc())
|
||||||
|
.limit(FOLDER_MAX)
|
||||||
|
).all()
|
||||||
|
return [
|
||||||
|
(job, asset)
|
||||||
|
for job, asset in rows
|
||||||
|
if storage.safe_abs_path(settings.files_root, asset.rel_path) is not None
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def is_expired(link: DownloadLink) -> bool:
|
def is_expired(link: DownloadLink) -> bool:
|
||||||
return link.expires_at is not None and link.expires_at <= datetime.now(timezone.utc)
|
return link.expires_at is not None and link.expires_at <= datetime.now(timezone.utc)
|
||||||
|
|
||||||
@@ -55,18 +115,22 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
|
|||||||
return False
|
return False
|
||||||
if exp < datetime.now(timezone.utc).timestamp():
|
if exp < datetime.now(timezone.utc).timestamp():
|
||||||
return False
|
return False
|
||||||
return hmac.compare_digest(sig, _sign(f"{token}.{exp}.{password_version}"))
|
# As bytes: compare_digest refuses (TypeError, a 500) a str with anything non-ASCII in it, and
|
||||||
|
# `sig` is whatever the URL carried.
|
||||||
|
return hmac.compare_digest(sig.encode(), _sign(f"{token}.{exp}.{password_version}").encode())
|
||||||
|
|
||||||
|
|
||||||
# --- serialization -------------------------------------------------------------------------
|
# --- serialization -------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def owner_view(link: DownloadLink, base_url: str = "") -> dict:
|
def owner_view(link: DownloadLink, base_url: str = "", token: str | None = None) -> dict:
|
||||||
"""What the file owner sees when managing their links."""
|
"""What the file owner sees when managing their links. `token` is the one just made (it is in
|
||||||
|
hand even where no copy is kept); else it is read back, and `url` is None when it cannot be."""
|
||||||
|
token = token or recover(link)
|
||||||
return {
|
return {
|
||||||
"id": link.id,
|
"id": link.id,
|
||||||
"token": link.token,
|
"token": token,
|
||||||
"url": f"{base_url}/watch/{link.token}",
|
"url": f"{base_url}/watch/{token}" if token else None,
|
||||||
"allow_download": link.allow_download,
|
"allow_download": link.allow_download,
|
||||||
"has_password": link.password_hash is not None,
|
"has_password": link.password_hash is not None,
|
||||||
"expires_at": link.expires_at.isoformat() if link.expires_at else None,
|
"expires_at": link.expires_at.isoformat() if link.expires_at else None,
|
||||||
|
|||||||
+37
-10
@@ -11,13 +11,12 @@ import html
|
|||||||
import re
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import links as linksmod
|
from app.downloads import links as linksmod
|
||||||
from app.downloads import storage
|
from app.downloads import storage
|
||||||
from app.models import DownloadJob, DownloadLink, MediaAsset
|
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset
|
||||||
|
|
||||||
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
|
_OG_BLOCK = re.compile(r"<!--OG:START-->.*?<!--OG:END-->", re.DOTALL)
|
||||||
|
|
||||||
@@ -74,22 +73,25 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
|
|||||||
|
|
||||||
def _watch_tags(token: str, db: Session) -> str | None:
|
def _watch_tags(token: str, db: Session) -> str | None:
|
||||||
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
|
"""Build the OG/Twitter meta block for a watch token, or None to keep the generic card."""
|
||||||
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
|
link = linksmod.find(db, token)
|
||||||
if link is None or linksmod.is_expired(link):
|
if link is None or linksmod.is_expired(link):
|
||||||
return None
|
return None
|
||||||
url = f"{settings.app_base}/watch/{token}"
|
url = f"{settings.app_base}/watch/{token}"
|
||||||
|
folder = link.folder_id is not None
|
||||||
if link.password_hash:
|
if link.password_hash:
|
||||||
# Password-gated: don't leak the title/thumbnail in the unfurl.
|
# Password-gated: don't leak the title/thumbnail in the unfurl.
|
||||||
return "\n ".join(
|
return "\n ".join(
|
||||||
[
|
[
|
||||||
"<title>Siftlode</title>",
|
"<title>Siftlode</title>",
|
||||||
_tag("og:title", "Password-protected video"),
|
_tag("og:title", "Password-protected " + ("folder" if folder else "video")),
|
||||||
_tag("og:description", "Shared via Siftlode"),
|
_tag("og:description", "Shared via Siftlode"),
|
||||||
_tag("og:type", "video.other"),
|
_tag("og:type", "website" if folder else "video.other"),
|
||||||
_tag("og:url", url),
|
_tag("og:url", url),
|
||||||
_tag("og:site_name", "Siftlode"),
|
_tag("og:site_name", "Siftlode"),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
if folder:
|
||||||
|
return _folder_tags(db, link, url)
|
||||||
job = db.get(DownloadJob, link.job_id)
|
job = db.get(DownloadJob, link.job_id)
|
||||||
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
||||||
if job is None or asset is None or asset.status != "ready":
|
if job is None or asset is None or asset.status != "ready":
|
||||||
@@ -135,7 +137,7 @@ def _watch_tags(token: str, db: Session) -> str | None:
|
|||||||
if image_dims is None and image.endswith("poster.jpg") and asset.poster_path:
|
if image_dims is None and image.endswith("poster.jpg") and asset.poster_path:
|
||||||
# safe_abs_path returns None if the poster file is missing (gc'd) or escapes the root —
|
# safe_abs_path returns None if the poster file is missing (gc'd) or escapes the root —
|
||||||
# guard it, else _jpeg_size(None) would raise an uncaught AttributeError on this public page.
|
# guard it, else _jpeg_size(None) would raise an uncaught AttributeError on this public page.
|
||||||
poster_abs = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
poster_abs = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||||
image_dims = _jpeg_size(poster_abs) if poster_abs else None
|
image_dims = _jpeg_size(poster_abs) if poster_abs else None
|
||||||
if image_dims:
|
if image_dims:
|
||||||
tags.append(_tag("og:image:width", str(image_dims[0])))
|
tags.append(_tag("og:image:width", str(image_dims[0])))
|
||||||
@@ -147,13 +149,38 @@ def _watch_tags(token: str, db: Session) -> str | None:
|
|||||||
return "\n ".join(tags)
|
return "\n ".join(tags)
|
||||||
|
|
||||||
|
|
||||||
def render_watch_html(token: str, index_html: Path, db: Session) -> str | None:
|
def _folder_tags(db: Session, link: DownloadLink, url: str) -> str | None:
|
||||||
"""Return index.html with the OG block replaced for this watch token, or None to serve the
|
"""A folder link's card: the folder's name and how many files it holds — never their names,
|
||||||
generic page unchanged (invalid/expired/not-ready link, or a template without the markers)."""
|
which the page shows only to whoever opens it."""
|
||||||
|
folder = db.get(FileFolder, link.folder_id)
|
||||||
|
if folder is None or not folder.name:
|
||||||
|
return None
|
||||||
|
count = len(linksmod.folder_files(db, folder)) # the page's own list, so the two agree
|
||||||
|
title = folder.name.strip()
|
||||||
|
about = f"{count} file{'' if count == 1 else 's'} · Shared via Siftlode"
|
||||||
|
return "\n ".join(
|
||||||
|
[
|
||||||
|
f"<title>{html.escape(title)}</title>",
|
||||||
|
_tag("og:title", title),
|
||||||
|
_tag("og:description", about),
|
||||||
|
_tag("og:type", "website"),
|
||||||
|
_tag("og:url", url),
|
||||||
|
_tag("og:site_name", "Siftlode"),
|
||||||
|
_tag("twitter:title", title, attr="name"),
|
||||||
|
_tag("twitter:description", about, attr="name"),
|
||||||
|
_tag("twitter:card", "summary", attr="name"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def render_watch_html(token: str, page_html: Path, db: Session) -> str | None:
|
||||||
|
"""Return the watch page (watch.html) with the OG block replaced for this token, or None to
|
||||||
|
serve the generic page unchanged (invalid/expired/not-ready link, or a template without the
|
||||||
|
markers)."""
|
||||||
tags = _watch_tags(token, db)
|
tags = _watch_tags(token, db)
|
||||||
if tags is None:
|
if tags is None:
|
||||||
return None
|
return None
|
||||||
text = index_html.read_text(encoding="utf-8")
|
text = page_html.read_text(encoding="utf-8")
|
||||||
if "<!--OG:START-->" not in text:
|
if "<!--OG:START-->" not in text:
|
||||||
return None
|
return None
|
||||||
# A function replacement avoids re.sub interpreting backslashes/group refs in `tags`.
|
# A function replacement avoids re.sub interpreting backslashes/group refs in `tags`.
|
||||||
|
|||||||
@@ -3,17 +3,20 @@
|
|||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import time
|
import time
|
||||||
from datetime import date, datetime, timedelta, timezone
|
from datetime import date, datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
|
from yt_dlp.utils import download_range_func
|
||||||
|
|
||||||
from app import logs, sysconfig
|
from app import logs, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
|
from app.downloads import clip as clipmod
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import errors as dlerrors
|
from app.downloads import errors as dlerrors
|
||||||
from app.downloads import adapters, credentials, formats, mediakind, probe, quota, service, storage
|
from app.downloads import adapters, credentials, formats, mediakind, playable, probe, quota
|
||||||
|
from app.downloads import service, storage
|
||||||
from app.downloads import urlguard
|
from app.downloads import urlguard
|
||||||
from app.downloads.jobdb import (
|
from app.downloads.jobdb import (
|
||||||
_edit_staging_dir,
|
_edit_staging_dir,
|
||||||
@@ -25,7 +28,9 @@ from app.downloads.jobdb import (
|
|||||||
)
|
)
|
||||||
from app.downloads.stated import StatedFailure
|
from app.downloads.stated import StatedFailure
|
||||||
from app.downloads.status import JobStatus
|
from app.downloads.status import JobStatus
|
||||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset, PlexItem
|
||||||
|
from app.plex import lookups as plex_lookups
|
||||||
|
from app.plex import paths as plex_paths
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
from app.titles import normalize_title
|
from app.titles import normalize_title
|
||||||
|
|
||||||
@@ -36,6 +41,15 @@ class _Aborted(Exception):
|
|||||||
"""Raised inside the progress hook when the job was paused/canceled mid-download."""
|
"""Raised inside the progress hook when the job was paused/canceled mid-download."""
|
||||||
|
|
||||||
|
|
||||||
|
class _ClipFetchFailed(Exception):
|
||||||
|
"""A clip's YouTube range fetch failed: its message and the download catalog's diagnosis, so a
|
||||||
|
clip on YouTube's wall says what a download of the same video would."""
|
||||||
|
|
||||||
|
def __init__(self, msg: str, diagnosis: dlerrors.Diagnosis):
|
||||||
|
super().__init__(msg)
|
||||||
|
self.diagnosis = diagnosis
|
||||||
|
|
||||||
|
|
||||||
class _OwnerRefused(Exception):
|
class _OwnerRefused(Exception):
|
||||||
"""A refusal that belongs to THIS job's owner, not to the source.
|
"""A refusal that belongs to THIS job's owner, not to the source.
|
||||||
|
|
||||||
@@ -58,6 +72,10 @@ _DB_WRITE_INTERVAL_S = 1.0
|
|||||||
_CANCEL_POLL_INTERVAL_S = 1.0
|
_CANCEL_POLL_INTERVAL_S = 1.0
|
||||||
|
|
||||||
_THUMB_SUFFIXES = (".jpg", ".jpeg", ".png", ".webp")
|
_THUMB_SUFFIXES = (".jpg", ".jpeg", ".png", ".webp")
|
||||||
|
# The job phase while a file is brought into the phone-safe shape (`_ensure_phone_safe`) for a
|
||||||
|
# download or an upload. Its own word, not yt-dlp's "processing": it reports ffmpeg's real progress,
|
||||||
|
# so the queue row shows a % for it, where yt-dlp's post-steps only get an indeterminate pulse.
|
||||||
|
CONVERTING = "converting"
|
||||||
# yt-dlp's working files in the staging dir. None of them is ever the finished media, but several
|
# yt-dlp's working files in the staging dir. None of them is ever the finished media, but several
|
||||||
# CAN be the biggest file there (a video-only per-format stream, an abandoned fragment run), so the
|
# CAN be the biggest file there (a video-only per-format stream, an abandoned fragment run), so the
|
||||||
# staging fallback must exclude them by name — picking one would store e.g. a silent video-only
|
# staging fallback must exclude them by name — picking one would store e.g. a silent video-only
|
||||||
@@ -75,18 +93,26 @@ def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *,
|
|||||||
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
||||||
silently while scheduler jobs and playlist-push already notified (feedback symmetry). A
|
silently while scheduler jobs and playlist-push already notified (feedback symmetry). A
|
||||||
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
||||||
scheduler._notify_done). `kind` ("download"/"edit") lets the inbox pick accurate copy; the
|
scheduler._notify_done). `kind` ("download"/"edit"/"upload") lets the inbox pick accurate copy; the
|
||||||
English title here is only a fallback for an unknown type. `user_id`/`title` are captured by the
|
English title here is only a fallback for an unknown type. `user_id`/`title` are captured by the
|
||||||
caller BEFORE its commit, so reading them here needs no post-commit refresh of an expired row."""
|
caller BEFORE its commit; only the folder is read here, once, for the notice's "Open"."""
|
||||||
if user_id is None:
|
if user_id is None:
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
|
# The notice's "Open" lands on the file in its folder: both ride along (a folder the file
|
||||||
|
# has left since is the page's to notice — it then looks among every file).
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
create_notification(
|
create_notification(
|
||||||
db,
|
db,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
type="download_done",
|
type="download_done",
|
||||||
title=f"Ready to watch: {title}" if title else "Ready to watch",
|
title=f"Ready to watch: {title}" if title else "Ready to watch",
|
||||||
data={"title": title, "kind": kind},
|
data={
|
||||||
|
"title": title,
|
||||||
|
"kind": kind,
|
||||||
|
"job_id": job_id,
|
||||||
|
"folder_id": job.folder_id if job else None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
@@ -208,7 +234,7 @@ def _cancel_poll(job_id: int, interval_s: float = _CANCEL_POLL_INTERVAL_S, *, no
|
|||||||
|
|
||||||
def cancelled() -> bool:
|
def cancelled() -> bool:
|
||||||
if not state["cancelled"] and allow():
|
if not state["cancelled"] and allow():
|
||||||
state["cancelled"] = _job_status(job_id) not in ("running", None)
|
state["cancelled"] = _job_status(job_id) not in (JobStatus.RUNNING, None)
|
||||||
return state["cancelled"]
|
return state["cancelled"]
|
||||||
|
|
||||||
return cancelled
|
return cancelled
|
||||||
@@ -328,7 +354,7 @@ def _media_info(info: dict) -> dict:
|
|||||||
|
|
||||||
|
|
||||||
def _extract_with_fallback(
|
def _extract_with_fallback(
|
||||||
job_id, asset_id, url, spec, staging, adapter, cookiefile=None, max_bytes=None
|
job_id, asset_id, url, spec, staging, adapter, cookiefile=None, max_bytes=None, section=None
|
||||||
):
|
):
|
||||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||||
@@ -342,9 +368,16 @@ def _extract_with_fallback(
|
|||||||
download as itself.
|
download as itself.
|
||||||
|
|
||||||
`cookiefile` is the acting user's per-host cookies.txt, already gated by
|
`cookiefile` is the acting user's per-host cookies.txt, already gated by
|
||||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated.
|
`credentials.resolve_cookiefile` (a clip: `resolve_clip_cookiefile`); passed to every attempt so
|
||||||
|
the fallback stays authenticated.
|
||||||
|
|
||||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt."""
|
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt. A
|
||||||
|
`section` fetch goes through yt-dlp's ffmpeg downloader, which does not read it: a clip holds
|
||||||
|
its own output to the cap (`_process_clip`).
|
||||||
|
|
||||||
|
`section` = (start_s, end_s) fetches only that range (an ad-hoc clip, see downloads/clip): the
|
||||||
|
range is stream-copied, and its first frame is the marked one (measured), so the clip's own
|
||||||
|
re-encode starts at 0 instead of re-fetching keyframes at the cut."""
|
||||||
client_sets = adapter.client_sets(authed=cookiefile is not None)
|
client_sets = adapter.client_sets(authed=cookiefile is not None)
|
||||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||||
# Resolving may read a page first — network time the stall watchdog must not see as a stall.
|
# Resolving may read a page first — network time the stall watchdog must not see as a stall.
|
||||||
@@ -370,6 +403,9 @@ def _extract_with_fallback(
|
|||||||
max_bytes=max_bytes,
|
max_bytes=max_bytes,
|
||||||
)
|
)
|
||||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||||
|
if section is not None:
|
||||||
|
opts["download_ranges"] = download_range_func([], [section])
|
||||||
|
opts["force_keyframes_at_cuts"] = False
|
||||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||||
# multi-client-set fallback would otherwise cross the stall timeout with a stale updated_at.
|
# multi-client-set fallback would otherwise cross the stall timeout with a stale updated_at.
|
||||||
# (Per-ATTEMPT only; a single extract_info that itself runs past the stall timeout is still
|
# (Per-ATTEMPT only; a single extract_info that itself runs past the stall timeout is still
|
||||||
@@ -406,58 +442,58 @@ def _extract_with_fallback(
|
|||||||
raise last_exc # unreachable (loop either returns or raises)
|
raise last_exc # unreachable (loop either returns or raises)
|
||||||
|
|
||||||
|
|
||||||
def _download_settings() -> tuple[str, int]:
|
def _download_layout() -> str:
|
||||||
"""Layout + retention days from the DB config (admin-overridable on the Configuration page),
|
"""The on-disk layout from the DB config (admin-overridable on the Configuration page), falling
|
||||||
falling back to the env defaults. Read here, not from `settings`, so an admin edit takes effect."""
|
back to the env default. Read here, not from `settings`, so an admin edit takes effect. (How long
|
||||||
|
a file stays is its folder's, stamped on the job when it is done — `folders.done_expiry`.)"""
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
return (
|
return sysconfig.get_str(db, "download_layout")
|
||||||
sysconfig.get_str(db, "download_layout"),
|
|
||||||
sysconfig.get_int(db, "download_retention_days"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict) -> Path:
|
def _ensure_phone_safe(
|
||||||
"""Make an mp4-compat-profile download play in every browser: H.264 + AAC/MP3 in mp4. If the
|
job_id: int,
|
||||||
download (whatever the source shipped) isn't already that, re-encode the offending stream(s) —
|
produced: Path,
|
||||||
or just remux when the codecs are fine but the container isn't. **Scope:** only the mp4 compat
|
out_dir: Path,
|
||||||
profile is normalized; an explicit `vcodec` or a non-mp4 custom profile is the user opting out
|
phase: str,
|
||||||
of the guarantee and is left as-is (so a `webm` profile can still ship VP9/Opus). Returns the
|
total_s: int | None = None,
|
||||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk.
|
*,
|
||||||
|
keep_source: bool = False,
|
||||||
|
) -> Path:
|
||||||
|
"""Bring a file that is about to become a ready library asset into the phone-safe shape
|
||||||
|
(`playable`), writing any converted copy into `out_dir`. Returns the file to store — `produced`
|
||||||
|
itself when it already is that shape, or isn't media at all — and removes `produced` once a
|
||||||
|
converted copy exists, unless `keep_source` (an upload's original, which must outlive the
|
||||||
|
conversion until the database points at the new file). The job shows `phase` with real ffmpeg
|
||||||
|
progress while it converts.
|
||||||
|
|
||||||
An audio download (mode "a") is never touched: yt-dlp already extracted it to .mp3 / .m4a, which
|
Every ready path calls this — a download, an upload's worker conversion and an editor clip — so
|
||||||
every browser plays as audio/mpeg / audio/mp4. Remuxing it into .mp4 served an mp3 as video/mp4,
|
a share link can only ever serve the one shape every phone and browser plays."""
|
||||||
which iPad's WebKit refuses to play (prod job 268)."""
|
p = playable.plan(probe.probe_json(produced), produced.suffix)
|
||||||
if spec.get("mode") == "a" or spec.get("container") != "mp4" or spec.get("vcodec"):
|
if p is None or not p.needs_change:
|
||||||
return produced
|
return produced
|
||||||
vcodec, acodec = editmod.probe_codecs(produced)
|
dest = out_dir / f"{produced.stem}.safe.{p.ext}"
|
||||||
is_mp4 = produced.suffix.lower() == ".mp4"
|
action = "re-encode" if (p.reencode_video or p.reencode_audio) else "remux"
|
||||||
needs, reenc_v, reenc_a = editmod.browser_transcode_flags(vcodec, acodec, is_mp4)
|
|
||||||
if not needs:
|
|
||||||
return produced
|
|
||||||
|
|
||||||
dest = produced.with_name(produced.stem + ".browser.mp4")
|
|
||||||
action = "re-encode" if (reenc_v or reenc_a) else "remux"
|
|
||||||
log.info(
|
log.info(
|
||||||
"job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4
|
"job %s: %s (video=%s audio=%s) -> phone-safe .%s",
|
||||||
|
job_id,
|
||||||
|
action,
|
||||||
|
p.reencode_video,
|
||||||
|
p.reencode_audio,
|
||||||
|
p.ext,
|
||||||
|
extra=logs.event("download.phone_safe", job_id=job_id, action=action, ext=p.ext),
|
||||||
)
|
)
|
||||||
_set_job(job_id, phase="processing", progress=0, speed_bps=None, eta_s=None)
|
_set_job(job_id, phase=phase, progress=0, speed_bps=None, eta_s=None)
|
||||||
# Prefer the info duration; probe the file when the extractor didn't provide one, so a long
|
# A re-encode with no known length would hold a frozen 0%; probe the file for a denominator.
|
||||||
# re-encode still shows real progress instead of a frozen 0%.
|
|
||||||
total_s = int(info["duration"]) if info.get("duration") else editmod.probe_duration(produced)
|
|
||||||
editmod.run_ffmpeg(
|
editmod.run_ffmpeg(
|
||||||
editmod.build_browser_safe_cmd(
|
playable.build_cmd(produced, dest, p, threads=editmod.encode_thread_cap()),
|
||||||
produced, dest, reenc_v, reenc_a, threads=editmod.encode_thread_cap()
|
total_s=total_s or editmod.probe_duration(produced),
|
||||||
),
|
on_progress=_ffmpeg_progress_writer(job_id, phase),
|
||||||
total_s=total_s,
|
|
||||||
on_progress=_ffmpeg_progress_writer(job_id, "processing"),
|
|
||||||
should_cancel=_cancel_poll(job_id),
|
should_cancel=_cancel_poll(job_id),
|
||||||
)
|
)
|
||||||
produced.unlink(missing_ok=True)
|
if not dest.exists():
|
||||||
# Keep the stored asset metadata truthful about what's now on disk.
|
raise StatedFailure("ffmpeg produced no output file")
|
||||||
if reenc_v:
|
if not keep_source:
|
||||||
info["vcodec"] = "h264"
|
produced.unlink(missing_ok=True)
|
||||||
if reenc_a:
|
|
||||||
info["acodec"] = "aac"
|
|
||||||
return dest
|
return dest
|
||||||
|
|
||||||
|
|
||||||
@@ -495,8 +531,8 @@ def _download(
|
|||||||
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
|
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
|
||||||
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a
|
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a
|
||||||
# single job could previously blow through the cap by any amount. Handing the remainder
|
# single job could previously blow through the cap by any amount. Handing the remainder
|
||||||
# to yt-dlp as `max_filesize` makes the format selector refuse an oversized file up
|
# to yt-dlp as `max_filesize` makes it refuse an oversized file (its HTTP downloader
|
||||||
# front instead.
|
# checks the announced size) instead.
|
||||||
remaining_bytes = quota.remaining_bytes(cdb, user_id)
|
remaining_bytes = quota.remaining_bytes(cdb, user_id)
|
||||||
# …and zero is a REFUSAL, not "no limit". Earlier jobs of this user may have filled the cap
|
# …and zero is a REFUSAL, not "no limit". Earlier jobs of this user may have filled the cap
|
||||||
# while this one sat in the queue, and 0 is not an expressible yt-dlp `max_filesize`, so
|
# while this one sat in the queue, and 0 is not an expressible yt-dlp `max_filesize`, so
|
||||||
@@ -505,8 +541,8 @@ def _download(
|
|||||||
# free space, which is what the `quota_exceeded` category tells them.
|
# free space, which is what the `quota_exceeded` category tells them.
|
||||||
if remaining_bytes == 0:
|
if remaining_bytes == 0:
|
||||||
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
|
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
|
||||||
info = _media_info(
|
with formats.watching_size_cap() as over_cap:
|
||||||
_extract_with_fallback(
|
fetched = _extract_with_fallback(
|
||||||
job_id,
|
job_id,
|
||||||
asset_id,
|
asset_id,
|
||||||
url,
|
url,
|
||||||
@@ -516,13 +552,30 @@ def _download(
|
|||||||
cookiefile=str(cookiefile) if cookiefile else None,
|
cookiefile=str(cookiefile) if cookiefile else None,
|
||||||
max_bytes=remaining_bytes,
|
max_bytes=remaining_bytes,
|
||||||
)
|
)
|
||||||
)
|
info = _media_info(dict(fetched))
|
||||||
|
# yt-dlp refused a file larger than what is left: it returns with no file, which would
|
||||||
|
# otherwise read as a processing fault. The owner's storage, so the owner's refusal.
|
||||||
|
if over_cap[0]:
|
||||||
|
raise _OwnerRefused(dlerrors.DOWNLOAD_TOO_BIG_MESSAGE)
|
||||||
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
||||||
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
||||||
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||||
|
|
||||||
produced = _produced_file(info, staging)
|
produced = _produced_file(info, staging)
|
||||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
produced = _ensure_phone_safe(
|
||||||
|
job_id,
|
||||||
|
produced,
|
||||||
|
staging,
|
||||||
|
CONVERTING,
|
||||||
|
int(info["duration"]) if info.get("duration") else None,
|
||||||
|
)
|
||||||
|
# Held to the cap by its real size, as a clip is: yt-dlp's `max_filesize` only stops a plain
|
||||||
|
# HTTP file that announces its size, and YouTube's fragmented formats never meet it
|
||||||
|
# (measured: 718 MB fetched with 1 MB left). Read again — other jobs may have landed.
|
||||||
|
with SessionLocal() as qdb:
|
||||||
|
room = quota.remaining_bytes(qdb, user_id)
|
||||||
|
if room is not None and produced.stat().st_size > room:
|
||||||
|
raise _OwnerRefused(dlerrors.DOWNLOAD_TOO_BIG_MESSAGE)
|
||||||
ext = produced.suffix.lstrip(".").lower()
|
ext = produced.suffix.lstrip(".").lower()
|
||||||
# Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's
|
# Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's
|
||||||
# `info` dict, whose fields are absent for some sources (Facebook among them): deriving
|
# `info` dict, whose fields are absent for some sources (Facebook among them): deriving
|
||||||
@@ -541,19 +594,18 @@ def _download(
|
|||||||
thumbnail_url=info.get("thumbnail"),
|
thumbnail_url=info.get("thumbnail"),
|
||||||
)
|
)
|
||||||
|
|
||||||
layout, retention_days = _download_settings()
|
layout = _download_layout()
|
||||||
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
|
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
|
||||||
thumb = _find_thumbnail(staging, produced)
|
thumb = _find_thumbnail(staging, produced)
|
||||||
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
|
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
|
||||||
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
|
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
|
||||||
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
|
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
|
||||||
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
|
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
|
||||||
storage.place_file(produced, settings.download_root, rel)
|
storage.place_file(produced, settings.files_root, rel)
|
||||||
nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, poster_jpg)
|
nfo_ok = storage.write_sidecars(settings.files_root, rel, meta, poster_jpg)
|
||||||
|
|
||||||
final = storage.abs_path(settings.download_root, rel)
|
final = storage.abs_path(settings.files_root, rel)
|
||||||
size = final.stat().st_size if final.exists() else None
|
size = final.stat().st_size if final.exists() else None
|
||||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
|
||||||
|
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
asset = db.get(MediaAsset, asset_id)
|
asset = db.get(MediaAsset, asset_id)
|
||||||
@@ -579,12 +631,11 @@ def _download(
|
|||||||
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
|
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
|
||||||
# and can't be relied on cross-origin. write_sidecars already wrote <base>.jpg from
|
# and can't be relied on cross-origin. write_sidecars already wrote <base>.jpg from
|
||||||
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
|
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
|
||||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||||
asset.source_webpage_url = info.get("webpage_url")
|
asset.source_webpage_url = info.get("webpage_url")
|
||||||
asset.nfo_written = nfo_ok
|
asset.nfo_written = nfo_ok
|
||||||
asset.error = None
|
asset.error = None
|
||||||
asset.last_access_at = datetime.now(timezone.utc)
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
asset.expires_at = expires
|
|
||||||
job = db.get(DownloadJob, job_id)
|
job = db.get(DownloadJob, job_id)
|
||||||
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
||||||
owner_id = job.user_id if job else None
|
owner_id = job.user_id if job else None
|
||||||
@@ -722,9 +773,19 @@ def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseExcep
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None = None) -> None:
|
def _fail_edit(
|
||||||
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
|
job_id: int,
|
||||||
where its line and its event code live. `exc` is the exception when there was one.
|
asset_id: int,
|
||||||
|
msg: str,
|
||||||
|
exc: BaseException | None = None,
|
||||||
|
*,
|
||||||
|
what: str = "edit",
|
||||||
|
diagnosis: dlerrors.Diagnosis | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Fail a local ffmpeg job — an edit, a clip, or an upload's conversion (`what="upload"`) — and
|
||||||
|
say so ONCE: every way one fails comes through here, so this is where its line and its event code
|
||||||
|
(`<what>.job.failed`) live. `exc` is the exception when there was one; `diagnosis` replaces the
|
||||||
|
plain category when the failure was a fetch the download catalog can name (a clip's).
|
||||||
|
|
||||||
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
|
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
|
||||||
Logged after the write, as on the download error path."""
|
Logged after the write, as on the download error path."""
|
||||||
@@ -739,19 +800,25 @@ def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None =
|
|||||||
job_id,
|
job_id,
|
||||||
status=JobStatus.ERROR,
|
status=JobStatus.ERROR,
|
||||||
error=msg,
|
error=msg,
|
||||||
error_code=dlerrors.classify(msg),
|
error_code=diagnosis.code
|
||||||
|
if diagnosis
|
||||||
|
else dlerrors.classify_conversion(msg)
|
||||||
|
if what == "upload"
|
||||||
|
else dlerrors.classify(msg),
|
||||||
|
error_detail=diagnosis.detail if diagnosis else None,
|
||||||
phase=None,
|
phase=None,
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
||||||
# built from the channel and the title.
|
# built from the channel and the title.
|
||||||
log.warning(
|
log.warning(
|
||||||
"edit job %s failed: %s",
|
"%s job %s failed: %s",
|
||||||
|
what,
|
||||||
job_id,
|
job_id,
|
||||||
formats.log_safe(msg),
|
formats.log_safe(msg),
|
||||||
extra={
|
extra={
|
||||||
**logs.event(
|
**logs.event(
|
||||||
"edit.job.failed",
|
f"{what}.job.failed",
|
||||||
job_id=job_id,
|
job_id=job_id,
|
||||||
asset_id=asset_id,
|
asset_id=asset_id,
|
||||||
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
||||||
@@ -777,7 +844,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
|||||||
if not src_ready or not src_rel:
|
if not src_ready or not src_rel:
|
||||||
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
||||||
return
|
return
|
||||||
src_path = storage.abs_path(settings.download_root, src_rel)
|
src_path = storage.abs_path(settings.files_root, src_rel)
|
||||||
if not src_path.exists():
|
if not src_path.exists():
|
||||||
_fail_edit(job_id, asset_id, "The source file is missing.")
|
_fail_edit(job_id, asset_id, "The source file is missing.")
|
||||||
return
|
return
|
||||||
@@ -811,13 +878,24 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
|||||||
)
|
)
|
||||||
if not dest_staging.exists():
|
if not dest_staging.exists():
|
||||||
raise StatedFailure("ffmpeg produced no output file")
|
raise StatedFailure("ffmpeg produced no output file")
|
||||||
|
# A fast trim stream-copies the source's codecs and container, so the clip is only as
|
||||||
|
# playable as its source: hold it to the same rule as every other ready file.
|
||||||
|
dest_staging = _ensure_phone_safe(job_id, dest_staging, staging, "editing", total_s)
|
||||||
|
out_ext = dest_staging.suffix.lstrip(".").lower() or out_ext
|
||||||
|
# New storage on the owner's footprint, held to what is left by its real size, as a
|
||||||
|
# download and a player clip are.
|
||||||
|
if user_id is not None:
|
||||||
|
with SessionLocal() as qdb:
|
||||||
|
room = quota.remaining_bytes(qdb, user_id)
|
||||||
|
if room is not None and dest_staging.stat().st_size > room:
|
||||||
|
raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||||
|
|
||||||
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||||
storage.place_file(dest_staging, settings.download_root, rel)
|
storage.place_file(dest_staging, settings.files_root, rel)
|
||||||
final = storage.abs_path(settings.download_root, rel)
|
final = storage.abs_path(settings.files_root, rel)
|
||||||
size = final.stat().st_size if final.exists() else None
|
size = final.stat().st_size if final.exists() else None
|
||||||
_, retention_days = _download_settings()
|
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
|
||||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
_, _, out_w, out_h, _ = probe.probe_media(final)
|
||||||
|
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
asset = db.get(MediaAsset, asset_id)
|
asset = db.get(MediaAsset, asset_id)
|
||||||
@@ -827,14 +905,15 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
|||||||
asset.size_bytes = size
|
asset.size_bytes = size
|
||||||
asset.container = out_ext
|
asset.container = out_ext
|
||||||
asset.duration_s = total_s or asset.duration_s
|
asset.duration_s = total_s or asset.duration_s
|
||||||
if crop:
|
if out_w and out_h:
|
||||||
|
asset.width, asset.height = out_w, out_h
|
||||||
|
elif crop:
|
||||||
asset.width = edit_spec["crop"]["w"]
|
asset.width = edit_spec["crop"]["w"]
|
||||||
asset.height = edit_spec["crop"]["h"]
|
asset.height = edit_spec["crop"]["h"]
|
||||||
else:
|
else:
|
||||||
asset.width, asset.height = src_w, src_h
|
asset.width, asset.height = src_w, src_h
|
||||||
asset.error = None
|
asset.error = None
|
||||||
asset.last_access_at = datetime.now(timezone.utc)
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
asset.expires_at = expires
|
|
||||||
job = db.get(DownloadJob, job_id)
|
job = db.get(DownloadJob, job_id)
|
||||||
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
|
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
|
||||||
owner_id = job.user_id if job else None
|
owner_id = job.user_id if job else None
|
||||||
@@ -867,3 +946,388 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
|||||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
|
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
|
||||||
finally:
|
finally:
|
||||||
shutil.rmtree(staging, ignore_errors=True)
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _clip_room(user_id: int) -> int | None:
|
||||||
|
"""What the clip's owner may still store (`quota.remaining_bytes`), refusing at zero — see
|
||||||
|
`_download`: zero is a refusal, not "no limit"."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
remaining = quota.remaining_bytes(db, user_id)
|
||||||
|
if remaining == 0:
|
||||||
|
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
|
||||||
|
return remaining
|
||||||
|
|
||||||
|
|
||||||
|
def _clip_source(
|
||||||
|
job_id: int, asset_id: int, spec: dict, staging: Path, remaining: int | None
|
||||||
|
) -> tuple[Path, float, str | None]:
|
||||||
|
"""The file a clip is cut from, where in it the clip starts, and the source's title when only
|
||||||
|
the fetch could tell it. A YouTube clip fetches ONLY its range into `staging` (so it starts at
|
||||||
|
0 there); a Plex clip reads the library file in place."""
|
||||||
|
if spec["source"] == "youtube":
|
||||||
|
url = service.source_url("youtube", spec["ref"])
|
||||||
|
cookiefile: Path | None = None
|
||||||
|
with SessionLocal() as db:
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
# Signed in when the user has a YouTube cookie, as their downloads are: YouTube gates
|
||||||
|
# single videos signed out ("not a bot"), and the clip is theirs alone.
|
||||||
|
if job and asset:
|
||||||
|
cookiefile = credentials.resolve_clip_cookiefile(
|
||||||
|
db, job.user_id, url, asset, spec, staging
|
||||||
|
)
|
||||||
|
_set_job(job_id, phase="downloading", progress=0)
|
||||||
|
try:
|
||||||
|
with formats.watching_size_cap() as over_cap:
|
||||||
|
info = _extract_with_fallback(
|
||||||
|
job_id,
|
||||||
|
asset_id,
|
||||||
|
url,
|
||||||
|
formats.normalize(clipmod.YOUTUBE_FETCH_SPEC),
|
||||||
|
staging,
|
||||||
|
adapters.resolve_adapter("youtube", spec["ref"]),
|
||||||
|
cookiefile=str(cookiefile) if cookiefile else None,
|
||||||
|
max_bytes=remaining,
|
||||||
|
section=(spec["start_s"], spec["end_s"]),
|
||||||
|
)
|
||||||
|
# Should yt-dlp's own cap ever refuse the range (see `_download`): the owner's storage.
|
||||||
|
if over_cap[0]:
|
||||||
|
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||||
|
except (_Aborted, _OwnerRefused):
|
||||||
|
raise
|
||||||
|
except Exception as exc: # re-raised below with the download catalog's diagnosis
|
||||||
|
msg = str(exc)[:500]
|
||||||
|
raise _ClipFetchFailed(
|
||||||
|
msg, dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
|
||||||
|
) from exc
|
||||||
|
finally:
|
||||||
|
# Never leave a plaintext session cookie on disk past the fetch (see `_download`).
|
||||||
|
if cookiefile is not None:
|
||||||
|
cookiefile.unlink(missing_ok=True)
|
||||||
|
info = _media_info(dict(info))
|
||||||
|
title = normalize_title(adapters.resolve_adapter("youtube", spec["ref"]).title(info))
|
||||||
|
return _produced_file(info, staging), 0.0, title
|
||||||
|
with SessionLocal() as db:
|
||||||
|
item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"])
|
||||||
|
src = plex_paths.local_media_path(db, item.file_path) if item else None
|
||||||
|
if src is None:
|
||||||
|
raise StatedFailure("The library file is not reachable from here.")
|
||||||
|
return src, spec["start_s"], None
|
||||||
|
|
||||||
|
|
||||||
|
def _clip_crop(spec: dict, data: dict) -> str:
|
||||||
|
"""The clip's crop filter, from its frame and the picture the file really holds ("" for none)."""
|
||||||
|
frame = spec.get("crop")
|
||||||
|
if not frame:
|
||||||
|
return ""
|
||||||
|
dar = clipmod.display_aspect(data)
|
||||||
|
if dar is None:
|
||||||
|
raise StatedFailure("The picture's size could not be read, so the crop cannot be placed.")
|
||||||
|
crop = clipmod.crop_filter(frame, dar)
|
||||||
|
if crop is None:
|
||||||
|
raise StatedFailure("The crop frame lies outside the picture.")
|
||||||
|
return crop
|
||||||
|
|
||||||
|
|
||||||
|
def _process_clip(job_id: int, asset_id: int) -> None:
|
||||||
|
"""Cut an ad-hoc clip (downloads/clip): fetch or open its source, ONE re-encode to the clip
|
||||||
|
shape, then into the user's library beside their edits."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
|
if job is None:
|
||||||
|
return
|
||||||
|
spec = dict(job.edit_spec or {})
|
||||||
|
user_id = job.user_id
|
||||||
|
staging = _edit_staging_dir(asset_id)
|
||||||
|
try:
|
||||||
|
if staging.exists():
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
staging.mkdir(parents=True, exist_ok=True)
|
||||||
|
# Refused before the fetch and the encode when the owner's storage is already full.
|
||||||
|
remaining = _clip_room(user_id)
|
||||||
|
src, start, fetched_title = _clip_source(job_id, asset_id, spec, staging, remaining)
|
||||||
|
data = probe.probe_json(src)
|
||||||
|
if not data:
|
||||||
|
raise StatedFailure("The source file could not be read.")
|
||||||
|
audio_track = spec.get("audio_track")
|
||||||
|
tracks = clipmod.audio_track_count(data)
|
||||||
|
if tracks == 0:
|
||||||
|
# A silent source: the player still names its track 0, and the clip is silent too.
|
||||||
|
audio_track = None
|
||||||
|
elif audio_track is not None and audio_track >= tracks:
|
||||||
|
raise StatedFailure("The chosen audio track is not in the file.")
|
||||||
|
crop = _clip_crop(spec, data)
|
||||||
|
total_s = clipmod.duration(spec)
|
||||||
|
out = staging / "clip.mp4"
|
||||||
|
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
||||||
|
editmod.run_ffmpeg(
|
||||||
|
clipmod.build_cmd(
|
||||||
|
src,
|
||||||
|
out,
|
||||||
|
start,
|
||||||
|
total_s,
|
||||||
|
audio_track=audio_track,
|
||||||
|
hdr=clipmod.is_hdr(data),
|
||||||
|
threads=editmod.encode_thread_cap(),
|
||||||
|
crop=crop,
|
||||||
|
),
|
||||||
|
total_s,
|
||||||
|
on_progress=_ffmpeg_progress_writer(job_id, "editing"),
|
||||||
|
should_cancel=_cancel_poll(job_id),
|
||||||
|
)
|
||||||
|
if not out.exists():
|
||||||
|
raise StatedFailure("ffmpeg produced no output file")
|
||||||
|
# The phone-safe shape by construction; still held to the rule every ready file meets.
|
||||||
|
out = _ensure_phone_safe(job_id, out, staging, "editing", round(total_s))
|
||||||
|
# Held to the cap by its real size: a range fetch passes yt-dlp's `max_filesize` to its
|
||||||
|
# ffmpeg downloader, which never reads it, and a Plex clip fetches nothing at all. Read
|
||||||
|
# again, since other jobs of this owner may have landed during the encode.
|
||||||
|
remaining = _clip_room(user_id)
|
||||||
|
if remaining is not None and out.stat().st_size > remaining:
|
||||||
|
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||||
|
ext = out.suffix.lstrip(".").lower() or "mp4"
|
||||||
|
rel = storage.edit_rel_path(user_id, asset_id, ext)
|
||||||
|
storage.place_file(out, settings.files_root, rel)
|
||||||
|
final = storage.abs_path(settings.files_root, rel)
|
||||||
|
vcodec, acodec, width, height, _ = probe.probe_media(final)
|
||||||
|
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
if asset:
|
||||||
|
asset.status = "ready"
|
||||||
|
asset.rel_path = rel
|
||||||
|
asset.size_bytes = final.stat().st_size if final.exists() else None
|
||||||
|
asset.container = ext
|
||||||
|
asset.vcodec, asset.acodec, asset.width, asset.height = (
|
||||||
|
vcodec,
|
||||||
|
acodec,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
)
|
||||||
|
asset.duration_s = round(total_s)
|
||||||
|
asset.media_kind = mediakind.derive(vcodec, acodec)
|
||||||
|
# The clip's own frame as its poster: the link preview shows the moment, and a Plex
|
||||||
|
# item has no public thumbnail at all.
|
||||||
|
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||||
|
asset.error = None
|
||||||
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
|
fields: dict = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
||||||
|
# A video outside the catalog was queued as "Clip (clip)": the fetch has its title now.
|
||||||
|
placeholder = service.clip_title(None, None)
|
||||||
|
if asset and fetched_title and asset.title == placeholder:
|
||||||
|
named = service.clip_title(fetched_title, None)
|
||||||
|
asset.title = named
|
||||||
|
if job_row := db.get(DownloadJob, job_id):
|
||||||
|
if job_row.display_name == placeholder:
|
||||||
|
fields["display_name"] = named[:DISPLAY_TEXT_MAX]
|
||||||
|
title = asset.title if asset else None
|
||||||
|
finalized = service.finalize_if_running(db, job_id, **fields)
|
||||||
|
db.commit()
|
||||||
|
if finalized:
|
||||||
|
_notify_download_done(db, user_id, title, job_id, kind="edit")
|
||||||
|
log.info(
|
||||||
|
"clip job %s done: asset %s",
|
||||||
|
job_id,
|
||||||
|
asset_id,
|
||||||
|
extra=logs.event(
|
||||||
|
"clip.job.done", job_id=job_id, asset_id=asset_id, source=spec["source"]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except (_Aborted, editmod.EditAborted):
|
||||||
|
# Paused/canceled mid-fetch or mid-encode: free the asset for a later retry.
|
||||||
|
_reset_asset_pending(asset_id)
|
||||||
|
log.info(
|
||||||
|
"clip job %s aborted (pause/cancel)",
|
||||||
|
job_id,
|
||||||
|
extra=logs.event("clip.job.aborted", job_id=job_id),
|
||||||
|
)
|
||||||
|
except _OwnerRefused as exc:
|
||||||
|
# Quota full: the job fails, the asset goes back to pending for a Retry once there is room
|
||||||
|
# (the download's handling; see `_download`).
|
||||||
|
msg = str(exc)[:500]
|
||||||
|
with SessionLocal() as db:
|
||||||
|
service.finalize_if_running(
|
||||||
|
db,
|
||||||
|
job_id,
|
||||||
|
status=JobStatus.ERROR,
|
||||||
|
error=msg,
|
||||||
|
error_code=dlerrors.classify(msg),
|
||||||
|
error_detail=None,
|
||||||
|
phase=None,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
_reset_asset_pending(asset_id)
|
||||||
|
log.info(
|
||||||
|
"clip job %s refused for its owner: %s",
|
||||||
|
job_id,
|
||||||
|
formats.log_safe(msg),
|
||||||
|
extra=logs.event("clip.job.refused", job_id=job_id),
|
||||||
|
)
|
||||||
|
except _ClipFetchFailed as exc:
|
||||||
|
# silent-ok: not silent — `_fail_edit` is the report, with the fetch's diagnosis.
|
||||||
|
_fail_edit(job_id, asset_id, str(exc), exc, what="clip", diagnosis=exc.diagnosis)
|
||||||
|
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||||
|
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||||
|
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="clip")
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _check_converted_fits(owner_id: int | None, old_size: int, new_size: int) -> None:
|
||||||
|
"""Refuse a converted upload that would push its owner over the storage cap: the upload was
|
||||||
|
charged at its original size, and a re-encode can come out larger. The original stays, counted,
|
||||||
|
for the owner to delete or keep."""
|
||||||
|
if owner_id is None:
|
||||||
|
return
|
||||||
|
with SessionLocal() as db:
|
||||||
|
if not quota.fits_after_conversion(db, owner_id, old_size, new_size):
|
||||||
|
raise StatedFailure(dlerrors.CONVERTED_TOO_BIG_MESSAGE)
|
||||||
|
|
||||||
|
|
||||||
|
def _unplace(placed: Path | None) -> None:
|
||||||
|
"""Remove a converted upload that was placed at its new rel but never committed: the row still
|
||||||
|
names the original, so nothing else would ever find this file."""
|
||||||
|
if placed is not None:
|
||||||
|
placed.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _conversion_landed(asset_id: int, new_rel: str) -> bool | None:
|
||||||
|
"""After the commit that makes a converted upload ready RAISED: did it land anyway? A commit can
|
||||||
|
reach the server and lose its answer (the connection drops after COMMIT). True/False by what the
|
||||||
|
row now says; None when the row cannot be read either."""
|
||||||
|
try:
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
return asset is not None and asset.status == "ready" and asset.rel_path == new_rel
|
||||||
|
except Exception: # noqa: BLE001 — "cannot tell" is the answer, the caller acts on it
|
||||||
|
# silent-ok: the caller re-raises the commit's own error, which `_fail_edit` reports.
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_original(job_id: int, src: Path) -> None:
|
||||||
|
"""Remove an upload's original once the converted file is committed in its place. Best effort:
|
||||||
|
the job is done and its file is ready, so a failure here must not fail it — the original is
|
||||||
|
named by no row now, and the GC's `.uploads` sweep takes it."""
|
||||||
|
try:
|
||||||
|
src.unlink(missing_ok=True)
|
||||||
|
except OSError as exc:
|
||||||
|
log.warning(
|
||||||
|
"upload job %s: original not removed, left to the GC sweep: %s",
|
||||||
|
job_id,
|
||||||
|
exc,
|
||||||
|
extra=logs.event("upload.original.unlink_failed", job_id=job_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _free_unheld_upload(asset_id: int) -> bool:
|
||||||
|
"""Free an upload no job holds any more — cancelled or deleted while its conversion ran, when
|
||||||
|
`release_asset` had to leave it because the conversion still read its file. Once that has
|
||||||
|
stopped, free it as the release would have; left alone, its file would sit on disk counted by no
|
||||||
|
owner's footprint until the GC's upload pass. False when a job still holds it (paused)."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
if asset is None or asset.ref_count != 0:
|
||||||
|
return False
|
||||||
|
if asset.rel_path:
|
||||||
|
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||||
|
db.delete(asset)
|
||||||
|
db.commit()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||||
|
"""Convert a queued upload into the phone-safe shape (`playable`) and make it ready.
|
||||||
|
|
||||||
|
The original sits at the asset's `rel_path` (see `upload`); the converted file replaces it there
|
||||||
|
under the new extension, and only then is the original removed — so a failed or stopped
|
||||||
|
conversion leaves the original in place for Retry, and a delete still finds it by `rel_path`.
|
||||||
|
A converted file placed but never committed is removed again on the way out. When the commit's
|
||||||
|
own outcome is unknown neither file is removed: the row names one of them, and the GC's `.uploads`
|
||||||
|
sweep takes the one it does not — as it does an original whose unlink after the commit failed."""
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
old_rel = asset.rel_path if asset else None
|
||||||
|
old_ext = asset.container if asset else None
|
||||||
|
old_size = (asset.size_bytes or 0) if asset else 0
|
||||||
|
duration = asset.duration_s if asset else None
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
|
owner_id = job.user_id if job else None
|
||||||
|
src = storage.safe_abs_path(settings.files_root, old_rel) if old_rel else None
|
||||||
|
if old_rel is None or src is None or not src.exists():
|
||||||
|
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
|
||||||
|
return
|
||||||
|
staging = _staging_dir(asset_id)
|
||||||
|
placed: Path | None = None # the converted file while it is at its new rel, uncommitted
|
||||||
|
try:
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
staging.mkdir(parents=True, exist_ok=True)
|
||||||
|
safe = _ensure_phone_safe(job_id, src, staging, CONVERTING, duration, keep_source=True)
|
||||||
|
ext = safe.suffix.lstrip(".").lower()
|
||||||
|
new_rel = storage.with_ext(old_rel, old_ext, ext)
|
||||||
|
if safe != src:
|
||||||
|
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
|
||||||
|
storage.place_file(safe, settings.files_root, new_rel)
|
||||||
|
if new_rel != old_rel:
|
||||||
|
placed = storage.abs_path(settings.files_root, new_rel)
|
||||||
|
final = storage.abs_path(settings.files_root, new_rel)
|
||||||
|
vcodec, acodec, width, height, dur = probe.probe_media(final)
|
||||||
|
with SessionLocal() as db:
|
||||||
|
asset = db.get(MediaAsset, asset_id)
|
||||||
|
if asset:
|
||||||
|
asset.status = "ready"
|
||||||
|
asset.rel_path = new_rel
|
||||||
|
asset.container = ext
|
||||||
|
asset.size_bytes = final.stat().st_size
|
||||||
|
asset.vcodec, asset.acodec = vcodec, acodec
|
||||||
|
asset.width, asset.height = width or asset.width, height or asset.height
|
||||||
|
asset.duration_s = dur or asset.duration_s
|
||||||
|
asset.error = None
|
||||||
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
|
title = asset.title if asset else None
|
||||||
|
finalized = service.finalize_if_running(
|
||||||
|
db, job_id, status=JobStatus.DONE, progress=100, phase=None
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
db.commit()
|
||||||
|
except Exception:
|
||||||
|
landed = _conversion_landed(asset_id, new_rel)
|
||||||
|
if landed is False:
|
||||||
|
raise # rolled back: the original is still the asset's file — fail as any step
|
||||||
|
# Landed, or cannot tell: the converted file may be the asset's file now, so it is
|
||||||
|
# never removed blind (the sweep takes whichever file no row names).
|
||||||
|
placed = None
|
||||||
|
if landed is None:
|
||||||
|
raise
|
||||||
|
db.rollback()
|
||||||
|
placed = None
|
||||||
|
if finalized:
|
||||||
|
_notify_download_done(db, owner_id, title, job_id, kind="upload")
|
||||||
|
if new_rel != old_rel:
|
||||||
|
_drop_original(job_id, src)
|
||||||
|
if not finalized:
|
||||||
|
_free_unheld_upload(asset_id) # finished after its job was cancelled or deleted
|
||||||
|
if finalized:
|
||||||
|
log.info(
|
||||||
|
"upload job %s converted: asset %s",
|
||||||
|
job_id,
|
||||||
|
asset_id,
|
||||||
|
extra=logs.event("upload.job.done", job_id=job_id, asset_id=asset_id, ext=ext),
|
||||||
|
)
|
||||||
|
except editmod.EditAborted:
|
||||||
|
# Paused/canceled mid-encode: the original is untouched. A paused job keeps it for its
|
||||||
|
# resume; one cancelled or deleted holds it no more, so it is freed now.
|
||||||
|
_unplace(placed)
|
||||||
|
if not _free_unheld_upload(asset_id):
|
||||||
|
_reset_asset_pending(asset_id)
|
||||||
|
log.info(
|
||||||
|
"upload job %s conversion aborted (pause/cancel)",
|
||||||
|
job_id,
|
||||||
|
extra=logs.event("upload.job.aborted", job_id=job_id),
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||||
|
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||||
|
_unplace(placed)
|
||||||
|
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="upload")
|
||||||
|
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
"""The phone-safe rule: what a library file must be so it plays wherever a share link lands.
|
||||||
|
|
||||||
|
A shared `/watch` link is opened by whoever got it, on whatever they hold: an iPhone's WebKit, an
|
||||||
|
Android Chrome, a desktop Firefox. The one shape every one of them plays:
|
||||||
|
|
||||||
|
* video → H.264, 8-bit 4:2:0 (`yuv420p`, or its full-range twin `yuvj420p`), AAC audio, in .mp4
|
||||||
|
* audio only → MP3 in .mp3, or AAC in .m4a
|
||||||
|
|
||||||
|
Measured on the owner's iPhone + iPad (2026-10-05): AV1, Matroska, HEVC tagged `hev1` and MP3
|
||||||
|
inside an .mp4 all fail there. VP9/WebM, HEVC `hvc1` and 10-bit H.264 played on those devices, but
|
||||||
|
Chrome and Firefox do not decode HEVC and browsers decode 10-bit / 4:4:4 H.264 only patchily, so the
|
||||||
|
rule is the shape every browser plays, not the shape one device plays.
|
||||||
|
|
||||||
|
Every file that becomes a ready library asset passes through here — a download, an upload (the
|
||||||
|
worker converts it, see `upload`), and an editor clip — so a link can only ever serve this shape.
|
||||||
|
`plan` says what (if anything) must change; `build_cmd` is the one ffmpeg command that changes it:
|
||||||
|
re-encode only the stream that fails, copy the rest, or just remux when only the container is wrong.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.downloads import edit as editmod
|
||||||
|
from app.downloads import probe
|
||||||
|
|
||||||
|
_SAFE_VCODEC = "h264"
|
||||||
|
_SAFE_PIX_FMTS = frozenset({"yuv420p", "yuvj420p"})
|
||||||
|
_VIDEO_EXT = "mp4"
|
||||||
|
# An audio-only file: the codec, and the extension it plays in. Anything else becomes AAC in .m4a.
|
||||||
|
_AUDIO_HOME = {"mp3": "mp3", "aac": "m4a"}
|
||||||
|
_AUDIO_FALLBACK_EXT = "m4a"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Plan:
|
||||||
|
"""What makes one file phone-safe. `ext` is the extension the safe file has (no dot)."""
|
||||||
|
|
||||||
|
ext: str
|
||||||
|
has_video: bool
|
||||||
|
reencode_video: bool
|
||||||
|
reencode_audio: bool
|
||||||
|
needs_change: bool
|
||||||
|
|
||||||
|
|
||||||
|
def plan(data: dict | None, ext: str) -> Plan | None:
|
||||||
|
"""The phone-safe plan for a file whose `probe.probe_json` is `data` and whose extension is
|
||||||
|
`ext`; None when it is not media at all (an unreadable probe, a still image, no audio or video
|
||||||
|
stream) — such a file is a download-only 'file' and never plays inline, so there is nothing to
|
||||||
|
make safe."""
|
||||||
|
if data is None:
|
||||||
|
return None
|
||||||
|
video, audio = probe.primary_streams(data)
|
||||||
|
ext = ext.lower().lstrip(".")
|
||||||
|
if video is not None:
|
||||||
|
rv = (video.get("codec_name") or "").lower() != _SAFE_VCODEC or (
|
||||||
|
video.get("pix_fmt") or ""
|
||||||
|
).lower() not in _SAFE_PIX_FMTS
|
||||||
|
ra = audio is not None and (audio.get("codec_name") or "").lower() != "aac"
|
||||||
|
return Plan(_VIDEO_EXT, True, rv, ra, rv or ra or ext != _VIDEO_EXT)
|
||||||
|
if audio is not None:
|
||||||
|
home = _AUDIO_HOME.get((audio.get("codec_name") or "").lower())
|
||||||
|
target = home or _AUDIO_FALLBACK_EXT
|
||||||
|
return Plan(target, False, False, home is None, home is None or ext != target)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def build_cmd(src: Path, dest: Path, p: Plan, threads: int | None = None) -> list[str]:
|
||||||
|
"""The ffmpeg command that writes `src` to `dest` (whose suffix is `p.ext`) in the phone-safe
|
||||||
|
shape: the failing stream(s) re-encoded with the package's shared encoder settings, the rest
|
||||||
|
stream-copied. An audio-only result drops any cover-art stream. `-progress pipe:1`
|
||||||
|
streams machine-readable progress; `threads` caps the encoder so a re-encode cannot claim every
|
||||||
|
core on the shared box (see the worker-child de-prioritization)."""
|
||||||
|
cmd = [
|
||||||
|
"ffmpeg",
|
||||||
|
"-y",
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-nostdin",
|
||||||
|
"-progress",
|
||||||
|
"pipe:1",
|
||||||
|
"-nostats",
|
||||||
|
*editmod.INPUT_GUARD,
|
||||||
|
"-i",
|
||||||
|
str(src),
|
||||||
|
]
|
||||||
|
# Exactly the two streams `plan` judged: `V` is a video stream that is not cover art. Nothing
|
||||||
|
# else rides along — a subtitle or data track the .mp4 muxer cannot take would fail the run.
|
||||||
|
if p.has_video:
|
||||||
|
cmd += ["-map", "0:V:0", "-map", "0:a:0?"]
|
||||||
|
if p.reencode_video:
|
||||||
|
cmd += ["-vf", editmod.EVEN_FRAME, *editmod.X264_ARGS]
|
||||||
|
else:
|
||||||
|
cmd += ["-c:v", "copy"]
|
||||||
|
else:
|
||||||
|
cmd += ["-map", "0:a:0"]
|
||||||
|
cmd += editmod.AAC_ARGS if p.reencode_audio else ["-c:a", "copy"]
|
||||||
|
if threads and threads > 0:
|
||||||
|
cmd += ["-threads", str(threads)]
|
||||||
|
if p.ext in ("mp4", "m4a"):
|
||||||
|
cmd += ["-movflags", "+faststart"]
|
||||||
|
cmd += [str(dest)]
|
||||||
|
return cmd
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
"""What a library file previews as — the single home of the preview allow-list.
|
||||||
|
|
||||||
|
A generic file (`media_kind == "file"`; video and audio have their players) previews by its stored
|
||||||
|
`container`, the upload's cleaned extension, and ONLY through this table: an image is shown as an
|
||||||
|
image, a PDF in the browser's own viewer, and a text format as plain text. Anything else has no
|
||||||
|
preview. The served Content-Type comes from here too, never from a guess on the name, and HTML / SVG
|
||||||
|
sit in the TEXT list on purpose: rendered on this origin, an uploaded page would run with the
|
||||||
|
reader's session. The route adds `CSP: sandbox` on top, so even a mislabelled file (a "png" that is
|
||||||
|
really a page) gets no script and no origin.
|
||||||
|
|
||||||
|
The serializer hands the kind to the client (`asset.preview`), so the library and the server can
|
||||||
|
never disagree on which files open a preview.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import codecs
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
from app.downloads import mediakind
|
||||||
|
|
||||||
|
PreviewKind = Literal["image", "pdf", "text"]
|
||||||
|
|
||||||
|
IMAGE_TYPES: dict[str, str] = {
|
||||||
|
"png": "image/png",
|
||||||
|
"jpg": "image/jpeg",
|
||||||
|
"jpeg": "image/jpeg",
|
||||||
|
"gif": "image/gif",
|
||||||
|
"webp": "image/webp",
|
||||||
|
"avif": "image/avif",
|
||||||
|
"bmp": "image/bmp",
|
||||||
|
}
|
||||||
|
|
||||||
|
PDF_TYPE = "application/pdf"
|
||||||
|
|
||||||
|
# Read as text: notes, data, subtitles, config and source. HTML and SVG belong here (see above).
|
||||||
|
TEXT_EXTS: frozenset[str] = frozenset(
|
||||||
|
{
|
||||||
|
"txt", "text", "md", "markdown", "log", "csv", "tsv", "json", "jsonl", "ndjson", "xml",
|
||||||
|
"yaml", "yml", "toml", "ini", "conf", "cfg", "properties", "env", "srt", "vtt", "ass", "ssa",
|
||||||
|
"sub", "nfo", "cue", "m3u", "m3u8", "html", "htm", "xhtml", "svg", "css", "scss", "less",
|
||||||
|
"js", "mjs", "cjs", "ts", "tsx", "jsx", "py", "rb", "php", "pl", "lua", "go", "rs", "java",
|
||||||
|
"kt", "kts", "swift", "c", "h", "cc", "cpp", "hpp", "cs", "sh", "bash", "zsh", "fish", "ps1",
|
||||||
|
"bat", "cmd", "sql", "r", "dart", "diff", "patch", "tex", "rst", "adoc", "gradle", "dockerfile",
|
||||||
|
}
|
||||||
|
) # fmt: skip
|
||||||
|
|
||||||
|
TEXT_TYPE = "text/plain; charset=utf-8"
|
||||||
|
|
||||||
|
# A text preview is the file's head: enough to read, never a whole log loaded into a tab.
|
||||||
|
TEXT_PREVIEW_BYTES = 1 << 20
|
||||||
|
# A NUL byte in the first this-many bytes means the "text" is not text.
|
||||||
|
BINARY_SNIFF_BYTES = 8192
|
||||||
|
|
||||||
|
|
||||||
|
def kind(media_kind: str | None, container: str | None) -> PreviewKind | None:
|
||||||
|
"""The preview a library file gets, or None (a media file plays instead; others download)."""
|
||||||
|
if media_kind != mediakind.FILE:
|
||||||
|
return None
|
||||||
|
ext = (container or "").lower()
|
||||||
|
if ext in IMAGE_TYPES:
|
||||||
|
return "image"
|
||||||
|
if ext == "pdf":
|
||||||
|
return "pdf"
|
||||||
|
if ext in TEXT_EXTS:
|
||||||
|
return "text"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def media_type(preview: PreviewKind, container: str | None) -> str:
|
||||||
|
"""The Content-Type a preview is served with: from this table, never guessed from the name."""
|
||||||
|
if preview == "image":
|
||||||
|
return IMAGE_TYPES[(container or "").lower()]
|
||||||
|
if preview == "pdf":
|
||||||
|
return PDF_TYPE
|
||||||
|
return TEXT_TYPE
|
||||||
|
|
||||||
|
|
||||||
|
# A text that is not UTF-8 is read as Central European Windows: what an old Hungarian subtitle or
|
||||||
|
# note is written in (its ő and ű are not in Latin-1).
|
||||||
|
FALLBACK_ENCODING = "cp1250"
|
||||||
|
_BOMS = (
|
||||||
|
(codecs.BOM_UTF8, "utf-8-sig"),
|
||||||
|
(codecs.BOM_UTF16_LE, "utf-16"),
|
||||||
|
(codecs.BOM_UTF16_BE, "utf-16"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def looks_binary(head: bytes) -> bool:
|
||||||
|
"""Whether the start of a "text" file is binary after all (a NUL in it)."""
|
||||||
|
return b"\x00" in head[:BINARY_SNIFF_BYTES]
|
||||||
|
|
||||||
|
|
||||||
|
def text_utf8(head: bytes) -> bytes | None:
|
||||||
|
"""A text file's head as UTF-8, the one charset the preview is served in; None when it is not text.
|
||||||
|
A byte-order mark names its encoding (UTF-16 is full of NULs, so it is read before the binary
|
||||||
|
check); else UTF-8 is kept as it is — a character cut at the size cap included, the reader shows
|
||||||
|
that one as a replacement mark — and anything else is read as `FALLBACK_ENCODING`."""
|
||||||
|
for bom, encoding in _BOMS:
|
||||||
|
if head.startswith(bom):
|
||||||
|
return head.decode(encoding, errors="replace").encode()
|
||||||
|
if looks_binary(head):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
head.decode("utf-8")
|
||||||
|
except UnicodeDecodeError as e: # silent-ok: not UTF-8 is an answer — read it as the fallback
|
||||||
|
if not (e.reason == "unexpected end of data" and e.end == len(head)):
|
||||||
|
return head.decode(FALLBACK_ENCODING, errors="replace").encode()
|
||||||
|
return head
|
||||||
@@ -34,6 +34,28 @@ _IMAGE_FORMATS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def primary_streams(data: dict) -> tuple[dict | None, dict | None]:
|
||||||
|
"""(video stream, audio stream) — the first of each kind — from a NON-None ffprobe JSON dict, by
|
||||||
|
the same reading the stored facts use: a still image / image-sequence container has neither (it
|
||||||
|
is a generic 'file'), and a video stream flagged `attached_pic` (an mp3/m4a's cover art) is not a
|
||||||
|
video track. The phone-safe rule (`playable`) judges exactly these two streams."""
|
||||||
|
fmt_names = {
|
||||||
|
n.strip() for n in ((data.get("format") or {}).get("format_name") or "").split(",")
|
||||||
|
}
|
||||||
|
if fmt_names & _IMAGE_FORMATS:
|
||||||
|
return None, None
|
||||||
|
video = audio = None
|
||||||
|
for stream in data.get("streams", []):
|
||||||
|
kind = stream.get("codec_type")
|
||||||
|
if kind == "video" and video is None:
|
||||||
|
if (stream.get("disposition") or {}).get("attached_pic"):
|
||||||
|
continue # cover art, not a video track
|
||||||
|
video = stream
|
||||||
|
elif kind == "audio" and audio is None:
|
||||||
|
audio = stream
|
||||||
|
return video, audio
|
||||||
|
|
||||||
|
|
||||||
def _facts_from_probe(
|
def _facts_from_probe(
|
||||||
data: dict,
|
data: dict,
|
||||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||||
@@ -45,19 +67,11 @@ def _facts_from_probe(
|
|||||||
fmt_names = {n.strip() for n in (fmt.get("format_name") or "").split(",")}
|
fmt_names = {n.strip() for n in (fmt.get("format_name") or "").split(",")}
|
||||||
if fmt_names & _IMAGE_FORMATS:
|
if fmt_names & _IMAGE_FORMATS:
|
||||||
return None, None, None, None, None
|
return None, None, None, None, None
|
||||||
|
video, audio = primary_streams(data)
|
||||||
vcodec = acodec = None
|
vcodec = video.get("codec_name") if video else None
|
||||||
width = height = None
|
width = video.get("width") if video else None
|
||||||
for stream in data.get("streams", []):
|
height = video.get("height") if video else None
|
||||||
kind = stream.get("codec_type")
|
acodec = audio.get("codec_name") if audio else None
|
||||||
if kind == "video" and vcodec is None:
|
|
||||||
if (stream.get("disposition") or {}).get("attached_pic"):
|
|
||||||
continue # cover art, not a video track
|
|
||||||
vcodec = stream.get("codec_name")
|
|
||||||
width = stream.get("width")
|
|
||||||
height = stream.get("height")
|
|
||||||
elif kind == "audio" and acodec is None:
|
|
||||||
acodec = stream.get("codec_name")
|
|
||||||
|
|
||||||
duration = None
|
duration = None
|
||||||
try:
|
try:
|
||||||
@@ -71,16 +85,29 @@ def _facts_from_probe(
|
|||||||
return vcodec, acodec, width, height, duration
|
return vcodec, acodec, width, height, duration
|
||||||
|
|
||||||
|
|
||||||
def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
def probe_json(path: Path) -> dict | None:
|
||||||
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
|
"""The full ffprobe JSON (streams + format) of `path`, or None when ffprobe could not read it —
|
||||||
or the file isn't playable media (an unprobeable file — or a still image — is treated as a generic
|
the one call both the stored facts (`facts`) and the phone-safe rule read from."""
|
||||||
non-media file, so it becomes a download-only 'file' rather than a broken 'video')."""
|
return editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
|
||||||
|
|
||||||
|
def facts(
|
||||||
|
data: dict | None,
|
||||||
|
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||||
|
"""(vcodec, acodec, width, height, duration_s) of a `probe_json` result; all None when it is None
|
||||||
|
(an unprobeable file is treated as a generic non-media file, so it becomes a download-only 'file'
|
||||||
|
rather than a broken 'video')."""
|
||||||
if data is None:
|
if data is None:
|
||||||
return None, None, None, None, None
|
return None, None, None, None, None
|
||||||
return _facts_from_probe(data)
|
return _facts_from_probe(data)
|
||||||
|
|
||||||
|
|
||||||
|
def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||||
|
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
|
||||||
|
or the file isn't playable media (a still image is a generic 'file', see `primary_streams`)."""
|
||||||
|
return facts(probe_json(path))
|
||||||
|
|
||||||
|
|
||||||
def resolve_media_facts(
|
def resolve_media_facts(
|
||||||
path: Path, info: dict
|
path: Path, info: dict
|
||||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||||
@@ -94,18 +121,18 @@ def resolve_media_facts(
|
|||||||
is authoritative — INCLUDING a deliberate None (cover-art-only audio, a still image). A per-field
|
is authoritative — INCLUDING a deliberate None (cover-art-only audio, a still image). A per-field
|
||||||
`probe or info` fallback would let `info`'s stale vcodec resurrect a video track the file doesn't
|
`probe or info` fallback would let `info`'s stale vcodec resurrect a video track the file doesn't
|
||||||
have (yt-dlp reports a progressive `vcodec` for an audio-mode download whose video it then strips),
|
have (yt-dlp reports a progressive `vcodec` for an audio-mode download whose video it then strips),
|
||||||
mislabelling an audio download as a broken in-app 'video'. Only when ffprobe could NOT read the
|
mislabelling an audio download as a broken in-app 'video'. When ffprobe could NOT read the file
|
||||||
file at all do we fall back to `info` (the pre-probe behaviour, a safety net); the cosmetic
|
at all, no codec is reported: the probe runs with the media demuxer whitelist, so a file it
|
||||||
dimensions and the duration still gap-fill from `info` in either case."""
|
cannot open is one the player and the editor cannot open either — and taking `info`'s codecs
|
||||||
|
for it would make it an editable 'video' whose bytes are whatever the remote site served. The
|
||||||
|
cosmetic dimensions and the duration still gap-fill from `info` in either case."""
|
||||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||||
info_v = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
|
||||||
info_a = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
|
||||||
info_w = int(info["width"]) if info.get("width") else None
|
info_w = int(info["width"]) if info.get("width") else None
|
||||||
info_h = int(info["height"]) if info.get("height") else None
|
info_h = int(info["height"]) if info.get("height") else None
|
||||||
info_d = int(info["duration"]) if info.get("duration") else None
|
info_d = int(info["duration"]) if info.get("duration") else None
|
||||||
if data is None:
|
if data is None:
|
||||||
# ffprobe could not read the file — fall back to the extractor's info entirely.
|
# Unreadable to the guarded probe: a download-only 'file' (no codecs → media_kind FILE).
|
||||||
return info_v, info_a, info_w, info_h, info_d
|
return None, None, info_w, info_h, info_d
|
||||||
pv, pa, pw, ph, pd = _facts_from_probe(data)
|
pv, pa, pw, ph, pd = _facts_from_probe(data)
|
||||||
# Probe ran → its codecs win outright (a None means "no such track", not "unknown"); info only
|
# Probe ran → its codecs win outright (a None means "no such track", not "unknown"); info only
|
||||||
# gap-fills the width/height/duration the probe couldn't read.
|
# gap-fills the width/height/duration the probe couldn't read.
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
"""Per-user download limits + footprint accounting.
|
"""Per-user download limits + footprint accounting.
|
||||||
|
|
||||||
A user's *footprint* is the total size of the ready assets their library references (cache hits
|
A user's *footprint* is the total size of the ready assets their library references (cache hits
|
||||||
count — it's "how much disk you're responsible for", not "how much you personally downloaded").
|
count — it's "how much disk you're responsible for", not "how much you personally downloaded"). An
|
||||||
|
upload counts from the moment it lands, ready or not: its original is on disk while its conversion
|
||||||
|
waits, and stays there when the conversion fails (for Retry) — bytes a ready-only sum never saw.
|
||||||
Limits come from a per-user DownloadQuota row if the admin set one, else the sysconfig defaults.
|
Limits come from a per-user DownloadQuota row if the admin set one, else the sysconfig defaults.
|
||||||
`unlimited` (e.g. the admin's own account) bypasses the byte cap.
|
`unlimited` (e.g. the admin's own account) bypasses the byte cap.
|
||||||
|
|
||||||
@@ -13,21 +15,30 @@ the footprint as it stood when the job was created and earlier jobs may have fil
|
|||||||
That is a per-job ceiling, not an aggregate ledger: concurrent jobs of one user each read the same
|
That is a per-job ceiling, not an aggregate ledger: concurrent jobs of one user each read the same
|
||||||
remainder (in-flight bytes are not `ready`, so `footprint` cannot see them), so a wave of
|
remainder (in-flight bytes are not `ready`, so `footprint` cannot see them), so a wave of
|
||||||
max_concurrent downloads can still land over the cap — the deliberate trade, since a download's size
|
max_concurrent downloads can still land over the cap — the deliberate trade, since a download's size
|
||||||
is unknown until it finishes. Edit jobs do not consult it at all (tracked as SEC-19).
|
is unknown until it finishes. CLIP jobs consult it before their fetch or encode and again with
|
||||||
|
their output's real size before it is placed. Edit jobs do not consult it at all (tracked as SEC-19).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import and_, func, or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import sysconfig
|
from app import sysconfig
|
||||||
|
from app.downloads.status import JobKind, JobStatus
|
||||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||||
|
|
||||||
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
||||||
# (error, canceled) don't count.
|
# (error, canceled) don't count.
|
||||||
_HOLDING = ("queued", "running", "paused", "done")
|
_HOLDING = (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED, JobStatus.DONE)
|
||||||
_RUNNING = ("queued", "running")
|
_RUNNING = (JobStatus.QUEUED, JobStatus.RUNNING)
|
||||||
|
# An asset whose file is on disk: a ready one, or an upload in any state (its original lands at
|
||||||
|
# ingest, see the module docstring). Every byte sum — a user's footprint, the GC's total-cap pass,
|
||||||
|
# the admin's storage view — reads it, so they all count the same disk.
|
||||||
|
ON_DISK = or_(
|
||||||
|
MediaAsset.status == "ready",
|
||||||
|
and_(MediaAsset.source_kind == "upload", MediaAsset.rel_path.is_not(None)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -56,7 +67,8 @@ def resolve(db: Session, user_id: int) -> QuotaLimits:
|
|||||||
# "at limit" checks — the worker's claim-window exclusion vs at_concurrency_limit — since a
|
# "at limit" checks — the worker's claim-window exclusion vs at_concurrency_limit — since a
|
||||||
# 0-limit user is "capped" with zero running jobs, which the SQL exclusion can't see. Keep
|
# 0-limit user is "capped" with zero running jobs, which the SQL exclusion can't see. Keep
|
||||||
# them consistent by never letting the effective limit drop below 1. (To BLOCK a user's
|
# them consistent by never letting the effective limit drop below 1. (To BLOCK a user's
|
||||||
# downloads, suspend them or set max_jobs=0, not max_concurrent=0.)
|
# downloads, suspend them — the worker leaves a suspended user's queued jobs unclaimed — or
|
||||||
|
# set max_jobs=0, not max_concurrent=0.)
|
||||||
return QuotaLimits(row.max_bytes, max(1, row.max_concurrent), row.max_jobs, row.unlimited)
|
return QuotaLimits(row.max_bytes, max(1, row.max_concurrent), row.max_jobs, row.unlimited)
|
||||||
return QuotaLimits(
|
return QuotaLimits(
|
||||||
max_bytes=sysconfig.get_int(db, "download_default_max_bytes"),
|
max_bytes=sysconfig.get_int(db, "download_default_max_bytes"),
|
||||||
@@ -67,12 +79,17 @@ def resolve(db: Session, user_id: int) -> QuotaLimits:
|
|||||||
|
|
||||||
|
|
||||||
def footprint(db: Session, user_id: int) -> int:
|
def footprint(db: Session, user_id: int) -> int:
|
||||||
"""Total bytes of the distinct ready assets this user's active jobs reference."""
|
"""Total bytes of the distinct assets this user's jobs hold on disk: the ready ones their active
|
||||||
|
jobs reference, and every upload of theirs — including one whose conversion failed, which keeps
|
||||||
|
its original (see the module docstring)."""
|
||||||
held_assets = (
|
held_assets = (
|
||||||
select(DownloadJob.asset_id)
|
select(DownloadJob.asset_id)
|
||||||
.where(
|
.where(
|
||||||
DownloadJob.user_id == user_id,
|
DownloadJob.user_id == user_id,
|
||||||
DownloadJob.status.in_(_HOLDING),
|
or_(
|
||||||
|
DownloadJob.status.in_(_HOLDING),
|
||||||
|
and_(DownloadJob.job_kind == JobKind.UPLOAD, DownloadJob.status == JobStatus.ERROR),
|
||||||
|
),
|
||||||
DownloadJob.asset_id.is_not(None),
|
DownloadJob.asset_id.is_not(None),
|
||||||
)
|
)
|
||||||
.distinct()
|
.distinct()
|
||||||
@@ -80,7 +97,7 @@ def footprint(db: Session, user_id: int) -> int:
|
|||||||
return int(
|
return int(
|
||||||
db.execute(
|
db.execute(
|
||||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||||
MediaAsset.id.in_(held_assets), MediaAsset.status == "ready"
|
MediaAsset.id.in_(held_assets), ON_DISK
|
||||||
)
|
)
|
||||||
).scalar()
|
).scalar()
|
||||||
or 0
|
or 0
|
||||||
@@ -111,7 +128,7 @@ def at_concurrency_limit(db: Session, user_id: int) -> bool:
|
|||||||
lim = resolve(db, user_id)
|
lim = resolve(db, user_id)
|
||||||
if lim.unlimited:
|
if lim.unlimited:
|
||||||
return False
|
return False
|
||||||
running = _count(db, user_id, ("running",))
|
running = _count(db, user_id, (JobStatus.RUNNING,))
|
||||||
return running >= lim.max_concurrent
|
return running >= lim.max_concurrent
|
||||||
|
|
||||||
|
|
||||||
@@ -160,6 +177,18 @@ def check_upload(db: Session, user_id: int, add_bytes: int) -> None:
|
|||||||
raise QuotaExceeded("max_bytes", lim.max_bytes, projected)
|
raise QuotaExceeded("max_bytes", lim.max_bytes, projected)
|
||||||
|
|
||||||
|
|
||||||
|
def fits_after_conversion(db: Session, user_id: int, old_bytes: int, new_bytes: int) -> bool:
|
||||||
|
"""Whether an upload converted from `old_bytes` (what it is charged at now) to `new_bytes` keeps
|
||||||
|
its owner within the cap. A re-encode can come out larger than the original, and nothing else
|
||||||
|
re-checks it once the upload was let in."""
|
||||||
|
if new_bytes <= old_bytes: # a smaller file never breaks a cap the larger one was let in under
|
||||||
|
return True
|
||||||
|
lim = resolve(db, user_id)
|
||||||
|
if lim.unlimited:
|
||||||
|
return True
|
||||||
|
return footprint(db, user_id) - old_bytes + new_bytes <= lim.max_bytes
|
||||||
|
|
||||||
|
|
||||||
def usage(db: Session, user_id: int) -> dict:
|
def usage(db: Session, user_id: int) -> dict:
|
||||||
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
|
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
|
||||||
lim = resolve(db, user_id)
|
lim = resolve(db, user_id)
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from app.config import settings
|
|||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import formats
|
from app.downloads import formats
|
||||||
|
from app.downloads.status import JobKind
|
||||||
from app.models import DownloadJob
|
from app.models import DownloadJob
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
@@ -95,12 +96,16 @@ def _run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id) ->
|
|||||||
(`cause=child_exit`)."""
|
(`cause=child_exit`)."""
|
||||||
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
||||||
# real work, and to keep the import graph obvious: runner is a thin entry over the pipeline.
|
# real work, and to keep the import graph obvious: runner is a thin entry over the pipeline.
|
||||||
from app.downloads.pipeline import _download, _process_edit
|
from app.downloads.pipeline import _convert_upload, _download, _process_clip, _process_edit
|
||||||
|
|
||||||
with formats.hiding(source_ref):
|
with formats.hiding(source_ref):
|
||||||
try:
|
try:
|
||||||
if job_kind == "edit":
|
if job_kind == JobKind.EDIT:
|
||||||
_process_edit(job_id, asset_id)
|
_process_edit(job_id, asset_id)
|
||||||
|
elif job_kind == JobKind.UPLOAD:
|
||||||
|
_convert_upload(job_id, asset_id)
|
||||||
|
elif job_kind == JobKind.CLIP:
|
||||||
|
_process_clip(job_id, asset_id)
|
||||||
else:
|
else:
|
||||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||||
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
||||||
|
|||||||
@@ -16,11 +16,22 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import adapters
|
from app.downloads import adapters
|
||||||
|
from app.downloads import clip as clipmod
|
||||||
from app.downloads import credentials
|
from app.downloads import credentials
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import formats, mediakind, quota, storage
|
from app.downloads import folders, formats, mediakind, quota, storage
|
||||||
from app.downloads.status import HOLDING, JobStatus
|
from app.downloads.status import HOLDING, JobKind, JobStatus
|
||||||
from app.models import DISPLAY_TEXT_MAX, Channel, DownloadJob, MediaAsset, Video
|
from app.models import (
|
||||||
|
DISPLAY_TEXT_MAX,
|
||||||
|
Channel,
|
||||||
|
DownloadJob,
|
||||||
|
MediaAsset,
|
||||||
|
PlexItem,
|
||||||
|
PlexShow,
|
||||||
|
Video,
|
||||||
|
)
|
||||||
|
from app.plex import lookups as plex_lookups
|
||||||
|
from app.plex import paths as plex_paths
|
||||||
|
|
||||||
|
|
||||||
class EditError(Exception):
|
class EditError(Exception):
|
||||||
@@ -38,12 +49,15 @@ def source_url(source_kind: str, source_ref: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def reference_url(job, asset) -> str | None:
|
def reference_url(job, asset) -> str | None:
|
||||||
"""The clean "downloaded from" URL for a job — shared by the private Downloads page and the public
|
"""The clean "downloaded from" URL for a job — shared by the Files library and the public
|
||||||
watch page so both surfaces always agree. None for edit derivatives (a clip's source is the user's
|
watch page so both surfaces always agree. None for edit derivatives (a clip's source is the user's
|
||||||
own earlier download, not a web page); else the canonical page URL yt-dlp recorded; else one derived
|
own earlier download, not a web page) and for a Plex clip (the library is not a public page); a
|
||||||
from source_kind+source_ref (covers queued/older rows before the worker fills it)."""
|
YouTube clip links its video at the clip's start; else the canonical page URL yt-dlp recorded; else
|
||||||
if job is None or job.job_kind == "edit":
|
one derived from source_kind+source_ref (covers queued/older rows before the worker fills it)."""
|
||||||
|
if job is None or job.job_kind == JobKind.EDIT:
|
||||||
return None
|
return None
|
||||||
|
if job.job_kind == JobKind.CLIP:
|
||||||
|
return clipmod.reference_url(job.edit_spec or {})
|
||||||
if asset is not None and asset.source_webpage_url:
|
if asset is not None and asset.source_webpage_url:
|
||||||
return asset.source_webpage_url
|
return asset.source_webpage_url
|
||||||
if job.source_kind in ("youtube", "url"):
|
if job.source_kind in ("youtube", "url"):
|
||||||
@@ -51,6 +65,23 @@ def reference_url(job, asset) -> str | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def library_left_behind(db: Session, files_root: str) -> bool:
|
||||||
|
"""True when the library names files and none of the newest of them is under `files_root` — a
|
||||||
|
fresh, empty volume mounted in the library's place (a compose file changed under an install, its
|
||||||
|
old volume left behind). The mount check cannot see that: the new volume is a real mount."""
|
||||||
|
rels = (
|
||||||
|
db.execute(
|
||||||
|
select(MediaAsset.rel_path)
|
||||||
|
.where(MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None))
|
||||||
|
.order_by(MediaAsset.id.desc())
|
||||||
|
.limit(20)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return bool(rels) and not any(storage.abs_path(files_root, r).exists() for r in rels if r)
|
||||||
|
|
||||||
|
|
||||||
def get_or_create_asset(
|
def get_or_create_asset(
|
||||||
db: Session, source_kind: str, source_ref: str, format_sig: str
|
db: Session, source_kind: str, source_ref: str, format_sig: str
|
||||||
) -> MediaAsset:
|
) -> MediaAsset:
|
||||||
@@ -63,7 +94,12 @@ def get_or_create_asset(
|
|||||||
MediaAsset.source_ref == source_ref,
|
MediaAsset.source_ref == source_ref,
|
||||||
MediaAsset.format_sig == format_sig,
|
MediaAsset.format_sig == format_sig,
|
||||||
)
|
)
|
||||||
asset = db.execute(stmt).scalar_one_or_none()
|
# Locked to the caller's commit: a release or the GC freeing this asset meanwhile (they lock it
|
||||||
|
# too) either waits for the new hold, or has deleted the row by the time this read returns — and
|
||||||
|
# the caller then gets a fresh one, never a job pointing at a row that is gone.
|
||||||
|
asset = db.execute(
|
||||||
|
stmt.with_for_update().execution_options(populate_existing=True)
|
||||||
|
).scalar_one_or_none()
|
||||||
if asset is not None:
|
if asset is not None:
|
||||||
# A previously-FAILED shared asset must be re-attempted for the new job about to hold it —
|
# A previously-FAILED shared asset must be re-attempted for the new job about to hold it —
|
||||||
# reset it to pending (+clear the error) so the worker actually re-downloads instead of
|
# reset it to pending (+clear the error) so the worker actually re-downloads instead of
|
||||||
@@ -180,16 +216,18 @@ def enqueue(
|
|||||||
profile_snapshot=spec,
|
profile_snapshot=spec,
|
||||||
display_name=display_name
|
display_name=display_name
|
||||||
or (asset.title[:DISPLAY_TEXT_MAX] if asset.status == "ready" and asset.title else None),
|
or (asset.title[:DISPLAY_TEXT_MAX] if asset.status == "ready" and asset.title else None),
|
||||||
status="queued",
|
status=JobStatus.QUEUED,
|
||||||
queue_pos=next_queue_pos(db),
|
queue_pos=next_queue_pos(db),
|
||||||
)
|
)
|
||||||
db.add(job)
|
db.add(job)
|
||||||
|
folders.place(db, job)
|
||||||
incr_ref_count(db, asset.id)
|
incr_ref_count(db, asset.id)
|
||||||
|
|
||||||
# Cache hit: the file already exists — no worker round-trip needed.
|
# Cache hit: the file already exists — no worker round-trip needed.
|
||||||
if asset.status == "ready":
|
if asset.status == "ready":
|
||||||
job.status = "done"
|
job.status = JobStatus.DONE
|
||||||
job.progress = 100
|
job.progress = 100
|
||||||
|
folders.stamp_done(db, job)
|
||||||
asset.last_access_at = datetime.now(timezone.utc)
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
@@ -197,7 +235,7 @@ def enqueue(
|
|||||||
return job
|
return job
|
||||||
|
|
||||||
|
|
||||||
def _clip_title(source_title: str | None, display_name: str | None) -> str:
|
def clip_title(source_title: str | None, display_name: str | None) -> str:
|
||||||
# Clipped to DISPLAY_TEXT_MAX, not to a literal: this feeds `DownloadJob.display_name` through
|
# Clipped to DISPLAY_TEXT_MAX, not to a literal: this feeds `DownloadJob.display_name` through
|
||||||
# `enqueue_edit`, the same column `display_text` guards. A machine-made title, so it is clipped
|
# `enqueue_edit`, the same column `display_text` guards. A machine-made title, so it is clipped
|
||||||
# rather than run through `display_text` — blank here means "no title", not None.
|
# rather than run through `display_text` — blank here means "no title", not None.
|
||||||
@@ -239,7 +277,7 @@ def enqueue_edit(
|
|||||||
ref = f"{user_id}:{src_asset.id}"[:512]
|
ref = f"{user_id}:{src_asset.id}"[:512]
|
||||||
asset = get_or_create_asset(db, "edit", ref, sig)
|
asset = get_or_create_asset(db, "edit", ref, sig)
|
||||||
if not asset.title:
|
if not asset.title:
|
||||||
asset.title = _clip_title(src_asset.title, display_name)
|
asset.title = clip_title(src_asset.title, display_name)
|
||||||
asset.uploader = src_asset.uploader
|
asset.uploader = src_asset.uploader
|
||||||
asset.thumbnail_url = src_asset.thumbnail_url
|
asset.thumbnail_url = src_asset.thumbnail_url
|
||||||
asset.upload_date = src_asset.upload_date
|
asset.upload_date = src_asset.upload_date
|
||||||
@@ -251,7 +289,7 @@ def enqueue_edit(
|
|||||||
job = DownloadJob(
|
job = DownloadJob(
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
asset_id=asset.id,
|
asset_id=asset.id,
|
||||||
job_kind="edit",
|
job_kind=JobKind.EDIT,
|
||||||
source_kind="edit",
|
source_kind="edit",
|
||||||
source_ref=ref,
|
source_ref=ref,
|
||||||
source_job_id=source_job.id,
|
source_job_id=source_job.id,
|
||||||
@@ -259,15 +297,106 @@ def enqueue_edit(
|
|||||||
edit_spec=spec,
|
edit_spec=spec,
|
||||||
profile_snapshot={},
|
profile_snapshot={},
|
||||||
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
|
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
|
||||||
status="queued",
|
status=JobStatus.QUEUED,
|
||||||
queue_pos=next_queue_pos(db),
|
queue_pos=next_queue_pos(db),
|
||||||
)
|
)
|
||||||
db.add(job)
|
db.add(job)
|
||||||
|
folders.place(db, job)
|
||||||
incr_ref_count(db, asset.id)
|
incr_ref_count(db, asset.id)
|
||||||
|
|
||||||
if asset.status == "ready": # identical clip already produced for this user → instant
|
if asset.status == "ready": # identical clip already produced for this user → instant
|
||||||
job.status = "done"
|
job.status = JobStatus.DONE
|
||||||
job.progress = 100
|
job.progress = 100
|
||||||
|
folders.stamp_done(db, job)
|
||||||
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
db.refresh(job)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def _clip_source_facts(db: Session, spec: dict) -> dict:
|
||||||
|
"""Title, uploader, thumbnail and length of what a clip is cut from, read from our own catalog.
|
||||||
|
Raises ClipError when a Plex item is unknown or its file cannot be read here — said at enqueue,
|
||||||
|
rather than as a job that fails in the queue a minute later."""
|
||||||
|
if spec["source"] == "youtube":
|
||||||
|
video = db.get(Video, spec["ref"])
|
||||||
|
if video is None: # not in the catalog: titled "Clip" until yt-dlp names it (worker)
|
||||||
|
return {"title": None, "uploader": None, "thumbnail_url": None, "length_s": None}
|
||||||
|
channel = db.get(Channel, video.channel_id)
|
||||||
|
return {
|
||||||
|
"title": video.title,
|
||||||
|
"uploader": channel.title if channel else None,
|
||||||
|
"thumbnail_url": video.thumbnail_url,
|
||||||
|
"length_s": video.duration_seconds,
|
||||||
|
}
|
||||||
|
item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"])
|
||||||
|
if item is None:
|
||||||
|
raise clipmod.ClipError("not_found")
|
||||||
|
if plex_paths.local_media_path(db, item.file_path) is None:
|
||||||
|
raise clipmod.ClipError("media_missing")
|
||||||
|
title = item.title
|
||||||
|
show = db.get(PlexShow, item.show_id) if item.show_id else None
|
||||||
|
if show is not None:
|
||||||
|
episode = (
|
||||||
|
f"S{item.season_number}·E{item.episode_number}"
|
||||||
|
if item.season_number is not None and item.episode_number is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
title = " — ".join(p for p in (show.title, episode, item.title) if p)
|
||||||
|
return {
|
||||||
|
"title": title,
|
||||||
|
"uploader": None,
|
||||||
|
# Plex art is served through our own authenticated routes, never a public URL; the worker
|
||||||
|
# cuts the clip's poster from the clip itself.
|
||||||
|
"thumbnail_url": None,
|
||||||
|
"length_s": item.duration_s,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def enqueue_clip(
|
||||||
|
db: Session, user_id: int, spec: dict, display_name: str | None = None
|
||||||
|
) -> DownloadJob:
|
||||||
|
"""Queue an ad-hoc clip (`clipmod.normalize`d spec) for `user_id`: a NEW per-user asset cut
|
||||||
|
straight from the source — a YouTube range or a Plex library file (see downloads/clip).
|
||||||
|
|
||||||
|
Raises ClipError (unknown or unreachable source, a range past its end) or quota.QuotaExceeded."""
|
||||||
|
facts = _clip_source_facts(db, spec)
|
||||||
|
# One second of grace: a mark set on the last frame may round past the catalog's whole seconds.
|
||||||
|
if facts["length_s"] and spec["end_s"] > facts["length_s"] + 1:
|
||||||
|
raise clipmod.ClipError("past_end")
|
||||||
|
|
||||||
|
quota.check_enqueue(db, user_id)
|
||||||
|
|
||||||
|
ref = clipmod.asset_ref(user_id, spec)[:512]
|
||||||
|
asset = get_or_create_asset(db, "clip", ref, clipmod.sig(spec))
|
||||||
|
if not asset.title:
|
||||||
|
asset.title = clip_title(facts["title"], display_name)
|
||||||
|
asset.uploader = facts["uploader"]
|
||||||
|
asset.thumbnail_url = facts["thumbnail_url"]
|
||||||
|
asset.duration_s = round(clipmod.duration(spec))
|
||||||
|
asset.media_kind = mediakind.VIDEO
|
||||||
|
|
||||||
|
job = DownloadJob(
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
job_kind=JobKind.CLIP,
|
||||||
|
source_kind="clip",
|
||||||
|
source_ref=ref,
|
||||||
|
edit_spec=spec,
|
||||||
|
profile_snapshot={},
|
||||||
|
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
|
||||||
|
status=JobStatus.QUEUED,
|
||||||
|
queue_pos=next_queue_pos(db),
|
||||||
|
)
|
||||||
|
db.add(job)
|
||||||
|
folders.place(db, job)
|
||||||
|
incr_ref_count(db, asset.id)
|
||||||
|
|
||||||
|
if asset.status == "ready": # the same range was already cut for this user → instant
|
||||||
|
job.status = JobStatus.DONE
|
||||||
|
job.progress = 100
|
||||||
|
folders.stamp_done(db, job)
|
||||||
asset.last_access_at = datetime.now(timezone.utc)
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
@@ -291,6 +420,8 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
|
|||||||
if not fields:
|
if not fields:
|
||||||
return False
|
return False
|
||||||
db.flush() # asset-then-job lock order: push pending asset writes before we lock the job row
|
db.flush() # asset-then-job lock order: push pending asset writes before we lock the job row
|
||||||
|
if fields.get("status") == JobStatus.DONE:
|
||||||
|
return _finalize_done(db, job_id, {**fields, "ready_at": datetime.now(timezone.utc)})
|
||||||
# A Core UPDATE with the guard as a WHERE — same atomic "only while still running" write as the
|
# A Core UPDATE with the guard as a WHERE — same atomic "only while still running" write as the
|
||||||
# rest of the codebase's 2.0 statements, no hand-built SQL string. synchronize_session=False: this
|
# rest of the codebase's 2.0 statements, no hand-built SQL string. synchronize_session=False: this
|
||||||
# is a terminal write, so we don't pay to sync the (about-to-be-discarded) ORM identity map. Bound
|
# is a terminal write, so we don't pay to sync the (about-to-be-discarded) ORM identity map. Bound
|
||||||
@@ -304,6 +435,41 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
|
|||||||
return result.rowcount > 0
|
return result.rowcount > 0
|
||||||
|
|
||||||
|
|
||||||
|
def _finalize_done(db: Session, job_id: int, fields: dict) -> bool:
|
||||||
|
"""`finalize_if_running` for a job that finished: done is when a file's retention starts, its
|
||||||
|
folder's unless the user kept it. The expiry is written only while the job is still where it was
|
||||||
|
read from — a move, a keep, or its folder's delete (which sends its files home) landing while the
|
||||||
|
folder's retention was read sends the read round again, never stamping a folder's retention on a
|
||||||
|
file now in another one."""
|
||||||
|
while True:
|
||||||
|
held = db.execute(
|
||||||
|
select(DownloadJob.folder_id, DownloadJob.kept, DownloadJob.status).where(
|
||||||
|
DownloadJob.id == job_id
|
||||||
|
)
|
||||||
|
).one_or_none()
|
||||||
|
if held is None or held.status != JobStatus.RUNNING:
|
||||||
|
return False
|
||||||
|
stamped = {
|
||||||
|
**fields,
|
||||||
|
"expires_at": folders.done_expiry(db, held.folder_id, held.kept),
|
||||||
|
"expiry_warned": False,
|
||||||
|
}
|
||||||
|
hit = db.execute(
|
||||||
|
update(DownloadJob)
|
||||||
|
.where(
|
||||||
|
DownloadJob.id == job_id,
|
||||||
|
DownloadJob.status == JobStatus.RUNNING.value,
|
||||||
|
DownloadJob.folder_id.is_not_distinct_from(held.folder_id),
|
||||||
|
DownloadJob.kept == held.kept,
|
||||||
|
)
|
||||||
|
.values(**stamped)
|
||||||
|
.returning(DownloadJob.id)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if hit is not None:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def release_asset(db: Session, job: DownloadJob) -> None:
|
def release_asset(db: Session, job: DownloadJob) -> None:
|
||||||
"""Drop this job's hold on its asset when it leaves a holding state. Once NO job holds the
|
"""Drop this job's hold on its asset when it leaves a holding state. Once NO job holds the
|
||||||
asset anymore, delete the file + row immediately — a deleted download should free its disk,
|
asset anymore, delete the file + row immediately — a deleted download should free its disk,
|
||||||
@@ -318,16 +484,24 @@ def release_asset(db: Session, job: DownloadJob) -> None:
|
|||||||
re-enqueuing the same (source,format) — get_or_create_asset would reset that row to `pending` and
|
re-enqueuing the same (source,format) — get_or_create_asset would reset that row to `pending` and
|
||||||
+1 it, and deleting it under that would FK-null the new job's asset (a lost update on ref_count).
|
+1 it, and deleting it under that would FK-null the new job's asset (a lost update on ref_count).
|
||||||
A ready asset is still freed at ref==0 (its file is the point); an orphaned errored row is cheap
|
A ready asset is still freed at ref==0 (its file is the point); an orphaned errored row is cheap
|
||||||
(no file) and gets reused+reset by the next enqueue, or reclaimed by a later GC pass."""
|
(no file) and gets reused+reset by the next enqueue, or reclaimed by a later GC pass.
|
||||||
|
|
||||||
|
An UPLOAD is the exception: it is never shared or re-enqueued (its ref is unique), and while it
|
||||||
|
waits for or failed its phone-safe conversion the original file already sits at `rel_path` — so
|
||||||
|
it is freed at ref==0 whatever its state, except mid-conversion (`downloading`: the worker child
|
||||||
|
still reads it; the GC upload pass takes it once the child has stopped)."""
|
||||||
if not job.asset_id or job.status not in HOLDING:
|
if not job.asset_id or job.status not in HOLDING:
|
||||||
return
|
return
|
||||||
asset = db.get(MediaAsset, job.asset_id)
|
asset = db.get(MediaAsset, job.asset_id)
|
||||||
if asset is None:
|
if asset is None:
|
||||||
return
|
return
|
||||||
new_count = decr_ref_count(db, asset.id)
|
new_count = decr_ref_count(db, asset.id)
|
||||||
if new_count == 0 and asset.status == "ready":
|
freeable = asset.status == "ready" or (
|
||||||
|
asset.source_kind == "upload" and asset.status != "downloading"
|
||||||
|
)
|
||||||
|
if new_count == 0 and freeable:
|
||||||
if asset.rel_path:
|
if asset.rel_path:
|
||||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||||
db.delete(asset)
|
db.delete(asset)
|
||||||
|
|
||||||
|
|
||||||
@@ -456,4 +630,6 @@ def redownload(db: Session, job: DownloadJob) -> None:
|
|||||||
# name would survive the rebuild).
|
# name would survive the rebuild).
|
||||||
job.display_name = None
|
job.display_name = None
|
||||||
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
|
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
|
||||||
|
job.expires_at = None # stamped again when the rebuild is done
|
||||||
|
job.expiry_warned = False
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|||||||
@@ -33,6 +33,27 @@ HOLDING = frozenset(
|
|||||||
{JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED, JobStatus.DONE, JobStatus.ERROR}
|
{JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED, JobStatus.DONE, JobStatus.ERROR}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class JobKind(StrEnum):
|
||||||
|
"""What a job produces, which decides the worker path that runs it and what the library may do
|
||||||
|
with the result (only a DOWNLOAD has a web source to fetch again)."""
|
||||||
|
|
||||||
|
DOWNLOAD = "download" # yt-dlp fetches a source into a shared, deduplicated asset
|
||||||
|
EDIT = "edit" # a per-user trim/crop of a file already in the library
|
||||||
|
UPLOAD = "upload" # a file the user sent; the worker only converts it
|
||||||
|
CLIP = (
|
||||||
|
"clip" # a per-user range of a YouTube video or a Plex item, cut straight from the source
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# A job kind with no web source to fetch again: Re-download refuses these.
|
||||||
|
NOT_REFETCHABLE = frozenset({JobKind.EDIT, JobKind.UPLOAD, JobKind.CLIP})
|
||||||
|
|
||||||
|
|
||||||
# States a route sets to STOP a running job. The worker's supervisor must step in (kill the child, free
|
# States a route sets to STOP a running job. The worker's supervisor must step in (kill the child, free
|
||||||
# the asset, leave the route's status intact) when it observes one of these on a job it is running.
|
# the asset, leave the route's status intact) when it observes one of these on a job it is running.
|
||||||
STOPPED_BY_ROUTE = frozenset({JobStatus.PAUSED, JobStatus.CANCELED})
|
STOPPED_BY_ROUTE = frozenset({JobStatus.PAUSED, JobStatus.CANCELED})
|
||||||
|
|
||||||
|
|
||||||
|
# A job that will not move again on its own: the queue lists it only briefly after it got here.
|
||||||
|
FINISHED = frozenset({JobStatus.DONE, JobStatus.CANCELED})
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ a Plex library using a metadata agent picks up rich info. Channel = "show", vide
|
|||||||
date-based — the community-standard mapping for YouTube-in-Plex.
|
date-based — the community-standard mapping for YouTube-in-Plex.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import unicodedata
|
import unicodedata
|
||||||
@@ -67,6 +68,57 @@ _FOLD_MAP = str.maketrans(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _mount_points(mountinfo: str) -> list[str]:
|
||||||
|
"""The mount points a /proc/<pid>/mountinfo table lists (its fifth field, octal escapes undone)."""
|
||||||
|
points = []
|
||||||
|
for line in mountinfo.splitlines():
|
||||||
|
fields = line.split(" ")
|
||||||
|
if len(fields) > 4:
|
||||||
|
points.append(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), fields[4]))
|
||||||
|
return points
|
||||||
|
|
||||||
|
|
||||||
|
def _in_container() -> bool:
|
||||||
|
return Path("/.dockerenv").exists() or Path("/run/.containerenv").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def _read_mountinfo() -> str | None:
|
||||||
|
try:
|
||||||
|
return Path("/proc/self/mountinfo").read_text()
|
||||||
|
# silent-ok: no mount table to read (not Linux) — the caller then cannot tell, and refuses nothing.
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def files_root_problem(
|
||||||
|
root: str, mountinfo: str | None = None, in_container: bool | None = None
|
||||||
|
) -> str | None:
|
||||||
|
"""Why `root` cannot be the files library's volume, or None when it can. In a container, a root
|
||||||
|
that is no mount of its own (nor inside one) is the image's empty directory: the library would
|
||||||
|
answer 404 for every file, and what a download writes would go when the container does — the
|
||||||
|
state an install is in when its compose file still mounts the library somewhere else.
|
||||||
|
|
||||||
|
Asked of the kernel's mount table, not by comparing devices: a storage driver that keeps the
|
||||||
|
container's root on the host's own disk (vfs, the fallback under Docker-in-LXC on older ZFS) gives
|
||||||
|
a mounted volume the same device as `/`. Outside a container a plain directory is a fine root."""
|
||||||
|
path = Path(root)
|
||||||
|
if not path.is_dir():
|
||||||
|
return f"{root} does not exist"
|
||||||
|
if in_container if in_container is not None else _in_container():
|
||||||
|
table = mountinfo if mountinfo is not None else _read_mountinfo()
|
||||||
|
if table is not None:
|
||||||
|
real = os.path.realpath(root)
|
||||||
|
mounted = any(
|
||||||
|
m != "/" and (real == m or real.startswith(m.rstrip("/") + "/"))
|
||||||
|
for m in _mount_points(table)
|
||||||
|
)
|
||||||
|
if not mounted:
|
||||||
|
return f"{root} is not a mounted volume (it is the container's own empty directory)"
|
||||||
|
if not os.access(path, os.W_OK):
|
||||||
|
return f"{root} is not writable"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _ascii_fold(text: str) -> str:
|
def _ascii_fold(text: str) -> str:
|
||||||
"""Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
|
"""Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
|
||||||
the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have
|
the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have
|
||||||
@@ -122,7 +174,7 @@ def display_filename(name: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | None = None) -> str:
|
def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | None = None) -> str:
|
||||||
"""Path of the media file relative to DOWNLOAD_ROOT (forward slashes).
|
"""Path of the media file relative to FILES_ROOT (forward slashes).
|
||||||
|
|
||||||
`asset_id` disambiguates the file. A MediaAsset is unique on (source_kind, source_ref,
|
`asset_id` disambiguates the file. A MediaAsset is unique on (source_kind, source_ref,
|
||||||
format_sig), so the SAME video downloaded in two formats is two distinct assets — but without
|
format_sig), so the SAME video downloaded in two formats is two distinct assets — but without
|
||||||
@@ -144,7 +196,7 @@ def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | No
|
|||||||
|
|
||||||
|
|
||||||
def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
|
def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
|
||||||
"""On-disk path (relative to DOWNLOAD_ROOT) for a phase-2 editor derivative.
|
"""On-disk path (relative to FILES_ROOT) for a phase-2 editor derivative.
|
||||||
|
|
||||||
Kept in a per-user `.edits/` tree — device-downloadable via the file endpoint, but NOT part of
|
Kept in a per-user `.edits/` tree — device-downloadable via the file endpoint, but NOT part of
|
||||||
the Plex library (these are user clips, not canonical episodes; there are no .nfo/poster
|
the Plex library (these are user clips, not canonical episodes; there are no .nfo/poster
|
||||||
@@ -161,20 +213,32 @@ def _clean_upload_ext(filename: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
|
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
|
||||||
"""On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
|
"""On-disk path (relative to FILES_ROOT) for a locally-uploaded file.
|
||||||
|
|
||||||
Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of
|
Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of
|
||||||
the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster
|
the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster
|
||||||
episode sidecars). The name is system-decided and bound to the asset id (`_a{id}` keeps two
|
episode sidecars). The name is system-decided and bound to the asset id (`_a{id}` keeps two
|
||||||
same-named uploads apart); the original filename is display metadata only. The extension is
|
same-named uploads apart); the original filename is display metadata only. The extension is
|
||||||
preserved (shell-safe, bounded) so the served file + its Content-Disposition carry a correct
|
preserved (shell-safe, bounded) so the served file + its Content-Disposition carry a correct
|
||||||
suffix; an extensionless upload stays extensionless."""
|
suffix; an extensionless upload stays extensionless.
|
||||||
stem = sanitize(Path(filename).stem, 80)
|
|
||||||
|
The stem holds no dot, so the last dot of the name is always the extension's: every sibling
|
||||||
|
derived by swapping it (`with_suffix` — the poster, the converted file) keeps `_a{id}`. A dotted
|
||||||
|
stem with no extension ('Holiday.2024.ä') once lost it, and two such uploads shared one file."""
|
||||||
|
stem = sanitize(Path(filename).stem, 80).replace(".", "_")
|
||||||
ext = _clean_upload_ext(filename)
|
ext = _clean_upload_ext(filename)
|
||||||
suffix = f".{ext}" if ext else ""
|
suffix = f".{ext}" if ext else ""
|
||||||
return f".uploads/{user_id}/{stem}_a{asset_id}{suffix}"
|
return f".uploads/{user_id}/{stem}_a{asset_id}{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def with_ext(rel: str, old_ext: str | None, ext: str) -> str:
|
||||||
|
"""`rel` with its extension — `old_ext`, the stored `container`, None for a bare name — swapped
|
||||||
|
for `ext`. Not `Path.with_suffix`: on a bare name whose stem holds a dot (an upload placed before
|
||||||
|
the stems lost their dots) that would take the stem's tail for the extension, and `_a{id}` with it."""
|
||||||
|
base = rel[: -len(old_ext) - 1] if old_ext and rel.endswith(f".{old_ext}") else rel
|
||||||
|
return f"{base}.{ext}"
|
||||||
|
|
||||||
|
|
||||||
def upload_ext(filename: str) -> str | None:
|
def upload_ext(filename: str) -> str | None:
|
||||||
"""The stored `container` for an uploaded file: its shell-safe lowercased extension, or None when
|
"""The stored `container` for an uploaded file: its shell-safe lowercased extension, or None when
|
||||||
the upload has no extension (a bare name). Shares `_clean_upload_ext` with `upload_rel_path` so the
|
the upload has no extension (a bare name). Shares `_clean_upload_ext` with `upload_rel_path` so the
|
||||||
@@ -230,32 +294,39 @@ def served_media_type(media_kind: str | None, container: str | None) -> str | No
|
|||||||
def download_filename(display_name: str, container: str | None, path: Path) -> str:
|
def download_filename(display_name: str, container: str | None, path: Path) -> str:
|
||||||
"""The Content-Disposition filename for a served download: the cleaned display name carrying a single
|
"""The Content-Disposition filename for a served download: the cleaned display name carrying a single
|
||||||
correct extension — the asset container, else the file's own suffix — without doubling it when the
|
correct extension — the asset container, else the file's own suffix — without doubling it when the
|
||||||
name already ends in that extension. Shared by the private download and the public watch endpoints."""
|
name already ends in that extension. A file with no extension at all is served without one —
|
||||||
|
never as `name.` (Windows drops a trailing dot, so the saved file would not be the one named).
|
||||||
|
Shared by the private download and the public watch endpoints."""
|
||||||
ext = container or path.suffix.lstrip(".")
|
ext = container or path.suffix.lstrip(".")
|
||||||
base = display_filename(display_name)
|
base = display_filename(display_name)
|
||||||
|
if not ext:
|
||||||
|
return base
|
||||||
if base.lower().endswith(f".{ext.lower()}"):
|
if base.lower().endswith(f".{ext.lower()}"):
|
||||||
base = base[: -(len(ext) + 1)]
|
base = base[: -(len(ext) + 1)]
|
||||||
return f"{base}.{ext}"
|
return f"{base}.{ext}"
|
||||||
|
|
||||||
|
|
||||||
def abs_path(download_root: str, rel: str) -> Path:
|
def abs_path(files_root: str, rel: str) -> Path:
|
||||||
return Path(download_root) / rel
|
return Path(files_root) / rel
|
||||||
|
|
||||||
|
|
||||||
def safe_abs_path(download_root: str, rel: str) -> Path | None:
|
def safe_abs_path(files_root: str, rel: str | None) -> Path | None:
|
||||||
"""The resolved absolute path for `rel`, but ONLY if it stays inside DOWNLOAD_ROOT and exists on
|
"""The resolved absolute path for `rel`, but ONLY if it stays inside FILES_ROOT and exists on
|
||||||
disk — otherwise None. Centralizes the path-traversal + existence guard every file-serving endpoint
|
disk — otherwise None (no `rel` at all included: an asset without a file has nothing to serve).
|
||||||
shares, so the containment check lives in exactly one place."""
|
Centralizes the path-traversal + existence guard every file-serving endpoint shares, so the
|
||||||
root = Path(download_root).resolve()
|
containment check lives in exactly one place."""
|
||||||
path = abs_path(download_root, rel).resolve()
|
if not rel:
|
||||||
|
return None
|
||||||
|
root = Path(files_root).resolve()
|
||||||
|
path = abs_path(files_root, rel).resolve()
|
||||||
if root not in path.parents or not path.exists():
|
if root not in path.parents or not path.exists():
|
||||||
return None
|
return None
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
def place_file(staging_file: Path, download_root: str, rel: str) -> None:
|
def place_file(staging_file: Path, files_root: str, rel: str) -> None:
|
||||||
"""Move a completed staging file into its final location under DOWNLOAD_ROOT."""
|
"""Move a completed staging file into its final location under FILES_ROOT."""
|
||||||
dest = abs_path(download_root, rel)
|
dest = abs_path(files_root, rel)
|
||||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
shutil.move(str(staging_file), str(dest))
|
shutil.move(str(staging_file), str(dest))
|
||||||
|
|
||||||
@@ -289,9 +360,9 @@ def _sidecar(media: Path, ext: str) -> Path:
|
|||||||
return media.with_suffix(ext)
|
return media.with_suffix(ext)
|
||||||
|
|
||||||
|
|
||||||
def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
|
def write_sidecars(files_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
|
||||||
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
||||||
media = abs_path(download_root, rel)
|
media = abs_path(files_root, rel)
|
||||||
try:
|
try:
|
||||||
_sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
|
_sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
|
||||||
if thumb_src and thumb_src.exists():
|
if thumb_src and thumb_src.exists():
|
||||||
@@ -307,9 +378,9 @@ def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Pat
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def delete_asset_files(download_root: str, rel: str) -> None:
|
def delete_asset_files(files_root: str, rel: str) -> None:
|
||||||
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
|
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
|
||||||
media = abs_path(download_root, rel)
|
media = abs_path(files_root, rel)
|
||||||
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
|
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
|
||||||
try:
|
try:
|
||||||
p.unlink()
|
p.unlink()
|
||||||
@@ -323,7 +394,7 @@ def delete_asset_files(download_root: str, rel: str) -> None:
|
|||||||
extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__),
|
extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__),
|
||||||
)
|
)
|
||||||
# Prune empty Season/Channel dirs up to (but not including) the root.
|
# Prune empty Season/Channel dirs up to (but not including) the root.
|
||||||
root = Path(download_root).resolve()
|
root = Path(files_root).resolve()
|
||||||
parent = media.parent
|
parent = media.parent
|
||||||
while parent != root and root in parent.parents:
|
while parent != root and root in parent.parents:
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -2,14 +2,23 @@
|
|||||||
|
|
||||||
A user-uploaded file joins the SAME library as a downloaded one, so it inherits the whole function
|
A user-uploaded file joins the SAME library as a downloaded one, so it inherits the whole function
|
||||||
set already built (metadata chips, share links, and — for media — in-app play/edit). Unlike a
|
set already built (metadata chips, share links, and — for media — in-app play/edit). Unlike a
|
||||||
download it bypasses yt-dlp AND the worker entirely: the file is already on the host, so ingest
|
download it bypasses yt-dlp: the file is already on the host, so ingest probes it and places it on
|
||||||
probes it, places it on disk, and creates an already-`ready` MediaAsset + `done` DownloadJob in one
|
disk. A file that already is the phone-safe shape (`playable`) — or is not media at all — becomes an
|
||||||
step — the same "the file exists, mark it done now" shortcut `service.enqueue` takes on a cache hit.
|
already-`ready` MediaAsset + `done` DownloadJob in one step, the same "the file exists, mark it done
|
||||||
|
now" shortcut `service.enqueue` takes on a cache hit.
|
||||||
|
|
||||||
|
Media that is NOT phone-safe (an iPhone's HEVC .mov, a VP9 .webm, an .mkv) is placed as it came and
|
||||||
|
queued: the asset stays `pending` and the job `queued`, and the download worker converts it
|
||||||
|
(`pipeline._convert_upload`) — so a share link never hands a friend a file their phone cannot play.
|
||||||
|
Only the converted file is kept (the owner's choice: one file per upload). A 10-bit HDR original
|
||||||
|
becomes 8-bit H.264 with its colour tags kept — still tagged HDR (HLG/PQ, BT.2020), not tone-mapped
|
||||||
|
to SDR (the owner's choice too). Until then the original sits at the asset's `rel_path`, which is
|
||||||
|
what a failed conversion's Retry starts from again, and what a delete removes
|
||||||
|
(`service.release_asset`).
|
||||||
|
|
||||||
Kinds (the invariant lives in `mediakind`): a probe with a real video stream -> video, audio-only
|
Kinds (the invariant lives in `mediakind`): a probe with a real video stream -> video, audio-only
|
||||||
-> audio, neither -> a generic 'file' (pdf/zip/docx/…) that is download-only and never rendered
|
-> audio, neither -> a generic 'file' (pdf/zip/docx/…) that is download-only and never rendered
|
||||||
inline (see the force-download serving in the file/watch routes). This module is the ingest seam a
|
inline (see the force-download serving in the file/watch routes).
|
||||||
future sub-function (e.g. convert) plugs a new adapter into, mirroring how it sidesteps the worker.
|
|
||||||
|
|
||||||
An upload does NOT get an `expires_at`: it's user-provided and may be irreplaceable (there's no
|
An upload does NOT get an `expires_at`: it's user-provided and may be irreplaceable (there's no
|
||||||
source to re-download), so it doesn't age out on the retention clock the way a re-fetchable download
|
source to re-download), so it doesn't age out on the retention clock the way a re-fetchable download
|
||||||
@@ -22,9 +31,12 @@ from pathlib import Path
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import audit
|
||||||
|
from app.audit import AuditAction
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import mediakind, probe, quota, service, storage
|
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
|
||||||
|
from app.downloads.status import JobKind, JobStatus
|
||||||
from app.models import DownloadJob, MediaAsset, display_text
|
from app.models import DownloadJob, MediaAsset, display_text
|
||||||
|
|
||||||
|
|
||||||
@@ -43,16 +55,35 @@ def ingest_upload(
|
|||||||
src_path: Path,
|
src_path: Path,
|
||||||
size: int,
|
size: int,
|
||||||
display_name: str | None = None,
|
display_name: str | None = None,
|
||||||
|
sha256: str | None = None,
|
||||||
|
folder_id: int | None = None,
|
||||||
|
via: str | None = None,
|
||||||
) -> DownloadJob:
|
) -> DownloadJob:
|
||||||
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return the `done`
|
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return its job —
|
||||||
job. `src_path` is MOVED into place on success (the caller stages it to a temp file first).
|
`done`, or `queued` for the worker's phone-safe conversion (see the module docstring). `src_path`
|
||||||
|
is MOVED into place on success (the caller stages it to a temp file first). `sha256` is the
|
||||||
|
hash of those bytes, when the caller took it on the way in; `folder_id` the folder it asked for
|
||||||
|
(`folders.upload_target`), else its Uploads folder; `via` what the audit row names as the doer
|
||||||
|
when it was not the user themselves (an API token).
|
||||||
|
|
||||||
|
Raises folders.FolderError when that folder is not one an upload may land in (or is gone).
|
||||||
|
|
||||||
Raises quota.QuotaExceeded (holdings/footprint cap) — checked BEFORE any disk write, so a
|
Raises quota.QuotaExceeded (holdings/footprint cap) — checked BEFORE any disk write, so a
|
||||||
rejected upload leaves nothing behind and the caller just cleans up its temp file."""
|
rejected upload leaves nothing behind and the caller just cleans up its temp file."""
|
||||||
quota.check_upload(db, user_id, size) # the file's size is known → charge it up front
|
quota.check_upload(db, user_id, size) # the file's size is known → charge it up front
|
||||||
|
if folder_id is not None:
|
||||||
|
# Refused before any work; held only once the job is about to name it (below), not through
|
||||||
|
# the probe and the poster, which a rename or delete of the folder would wait out.
|
||||||
|
folders.upload_target(db, user_id, folder_id, hold=False)
|
||||||
|
|
||||||
vcodec, acodec, width, height, duration = probe.probe_media(src_path)
|
data = probe.probe_json(src_path)
|
||||||
|
vcodec, acodec, width, height, duration = probe.facts(data)
|
||||||
kind = mediakind.derive(vcodec, acodec)
|
kind = mediakind.derive(vcodec, acodec)
|
||||||
|
# Judged by the extension the file is PLACED under (and the worker re-judges), not the raw one:
|
||||||
|
# 'clip.mp4~' is stored as .mp4, and judging 'mp4~' queued a conversion with nothing to convert.
|
||||||
|
ext = storage.upload_ext(filename) or ""
|
||||||
|
safe = playable.plan(data, ext) if kind != mediakind.FILE else None
|
||||||
|
convert = safe is not None and safe.needs_change
|
||||||
|
|
||||||
# An upload never dedups (each file is its own asset): a random ref makes the cache key unique,
|
# An upload never dedups (each file is its own asset): a random ref makes the cache key unique,
|
||||||
# so `format_sig` is a constant marker rather than a format hash.
|
# so `format_sig` is a constant marker rather than a format hash.
|
||||||
@@ -64,19 +95,23 @@ def ingest_upload(
|
|||||||
db.flush() # need asset.id for the on-disk path
|
db.flush() # need asset.id for the on-disk path
|
||||||
|
|
||||||
rel = storage.upload_rel_path(user_id, asset.id, filename)
|
rel = storage.upload_rel_path(user_id, asset.id, filename)
|
||||||
storage.place_file(src_path, settings.download_root, rel)
|
storage.place_file(src_path, settings.files_root, rel)
|
||||||
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
|
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
|
||||||
# asset/job rows roll back but the placed file would be ORPHANED — nothing references it, the
|
# asset/job rows roll back and nothing names the placed file: the per-user delete path keys off
|
||||||
# per-user delete path keys off asset.rel_path (which is gone), and the GC orphan sweep skips the
|
# asset.rel_path (which is gone), and the GC's `.uploads` sweep reaches it only an hour later. So
|
||||||
# dot-prefixed `.uploads/` tree. So roll back the on-disk side too on any failure.
|
# roll back the on-disk side too — unless the commit itself raised after it landed (see below).
|
||||||
|
asset_id = asset.id
|
||||||
try:
|
try:
|
||||||
final = storage.abs_path(settings.download_root, rel)
|
final = storage.abs_path(settings.files_root, rel)
|
||||||
actual_size = final.stat().st_size if final.exists() else size
|
actual_size = final.stat().st_size if final.exists() else size
|
||||||
|
|
||||||
asset.status = "ready"
|
asset.status = "pending" if convert else "ready"
|
||||||
asset.rel_path = rel
|
asset.rel_path = rel
|
||||||
|
# The hash names the bytes that arrived: a conversion replaces them, so it is kept only
|
||||||
|
# when the file stays as it came.
|
||||||
|
asset.sha256 = None if convert else sha256
|
||||||
asset.size_bytes = actual_size
|
asset.size_bytes = actual_size
|
||||||
asset.container = storage.upload_ext(filename)
|
asset.container = ext or None
|
||||||
asset.vcodec = vcodec
|
asset.vcodec = vcodec
|
||||||
asset.acodec = acodec
|
asset.acodec = acodec
|
||||||
asset.width = width
|
asset.width = width
|
||||||
@@ -90,25 +125,44 @@ def ingest_upload(
|
|||||||
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
|
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
|
||||||
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
|
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
|
||||||
# server side even for a codec the browser can't play).
|
# server side even for a codec the browser can't play).
|
||||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||||
|
|
||||||
job = DownloadJob(
|
job = DownloadJob(
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
asset_id=asset.id,
|
asset_id=asset.id,
|
||||||
job_kind="upload",
|
job_kind=JobKind.UPLOAD,
|
||||||
source_kind="upload",
|
source_kind="upload",
|
||||||
source_ref=ref,
|
source_ref=ref,
|
||||||
profile_snapshot={},
|
profile_snapshot={},
|
||||||
display_name=display_text(display_name),
|
display_name=display_text(display_name),
|
||||||
status="done",
|
status=JobStatus.QUEUED if convert else JobStatus.DONE,
|
||||||
progress=100,
|
progress=0 if convert else 100,
|
||||||
queue_pos=service.next_queue_pos(db),
|
queue_pos=service.next_queue_pos(db),
|
||||||
)
|
)
|
||||||
db.add(job)
|
db.add(job)
|
||||||
|
folders.place(db, job)
|
||||||
|
if folder_id is not None:
|
||||||
|
job.folder_id = folders.upload_target(db, user_id, folder_id).id
|
||||||
|
if not convert:
|
||||||
|
folders.stamp_done(db, job)
|
||||||
service.incr_ref_count(db, asset.id)
|
service.incr_ref_count(db, asset.id)
|
||||||
|
db.flush()
|
||||||
|
audit.record(
|
||||||
|
db,
|
||||||
|
user_id,
|
||||||
|
AuditAction.FILE_UPLOAD,
|
||||||
|
target_type="file",
|
||||||
|
target_id=job.id,
|
||||||
|
summary=via,
|
||||||
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
|
if _row_landed(db, asset_id) is not False:
|
||||||
|
# The commit reached the server and lost its answer, or the row cannot be read: the file
|
||||||
|
# may be the asset's now, so it is never removed blind — the sweep takes it if no row
|
||||||
|
# names it.
|
||||||
|
raise
|
||||||
# The rows are gone; remove the now-unreferenced file so no bytes leak. Clean the poster by its
|
# The rows are gone; remove the now-unreferenced file so no bytes leak. Clean the poster by its
|
||||||
# DETERMINISTIC path (rel with a .jpg suffix, the same path ensure_poster writes to) rather than
|
# DETERMINISTIC path (rel with a .jpg suffix, the same path ensure_poster writes to) rather than
|
||||||
# ensure_poster's return value: it writes the frame directly to that path and returns None on an
|
# ensure_poster's return value: it writes the frame directly to that path and returns None on an
|
||||||
@@ -116,7 +170,17 @@ def ingest_upload(
|
|||||||
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
|
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
|
||||||
# when none was generated (audio/file, or a video whose poster never started).
|
# when none was generated (audio/file, or a video whose poster never started).
|
||||||
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
|
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
|
||||||
storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True)
|
storage.abs_path(settings.files_root, leaked).unlink(missing_ok=True)
|
||||||
raise
|
raise
|
||||||
db.refresh(job)
|
db.refresh(job)
|
||||||
return job
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def _row_landed(db: Session, asset_id: int) -> bool | None:
|
||||||
|
"""After the ingest transaction raised: is the asset row there anyway? A commit can reach the
|
||||||
|
server and lose its answer (the connection drops after COMMIT). None when it cannot be read."""
|
||||||
|
try:
|
||||||
|
return db.get(MediaAsset, asset_id) is not None
|
||||||
|
except Exception: # noqa: BLE001 — "cannot tell" is the answer, the caller acts on it
|
||||||
|
# silent-ok: the caller re-raises the ingest's own error.
|
||||||
|
return None
|
||||||
|
|||||||
+3
-1
@@ -690,7 +690,9 @@ _POLLED_ROUTES = frozenset(
|
|||||||
"/api/me/notifications/unread_count",
|
"/api/me/notifications/unread_count",
|
||||||
"/api/downloads",
|
"/api/downloads",
|
||||||
"/api/downloads/index",
|
"/api/downloads/index",
|
||||||
"/api/downloads/usage",
|
"/api/files",
|
||||||
|
"/api/files/folders",
|
||||||
|
"/api/files/usage",
|
||||||
"/api/sync/my-status",
|
"/api/sync/my-status",
|
||||||
"/api/admin/scheduler",
|
"/api/admin/scheduler",
|
||||||
"/api/admin/downloads",
|
"/api/admin/downloads",
|
||||||
|
|||||||
+35
-6
@@ -28,6 +28,8 @@ log = logs.get_logger(__name__)
|
|||||||
from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first
|
from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first
|
||||||
from app.config import settings # noqa: E402 — logging is configured first
|
from app.config import settings # noqa: E402 — logging is configured first
|
||||||
from app.db import SessionLocal, engine # noqa: E402 — logging is configured first
|
from app.db import SessionLocal, engine # noqa: E402 — logging is configured first
|
||||||
|
from app.downloads import service as download_service # noqa: E402 — logging is configured first
|
||||||
|
from app.downloads import storage # noqa: E402 — logging is configured first
|
||||||
from app.plex import stream as plex_stream # noqa: E402 — logging is configured first
|
from app.plex import stream as plex_stream # noqa: E402 — logging is configured first
|
||||||
from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first
|
from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first
|
||||||
from app.routes import ( # noqa: E402 — logging is configured first
|
from app.routes import ( # noqa: E402 — logging is configured first
|
||||||
@@ -42,6 +44,7 @@ from app.routes import ( # noqa: E402 — logging is configured first
|
|||||||
discover as discover_routes,
|
discover as discover_routes,
|
||||||
downloads,
|
downloads,
|
||||||
feed,
|
feed,
|
||||||
|
files as files_routes,
|
||||||
health,
|
health,
|
||||||
me,
|
me,
|
||||||
notifications,
|
notifications,
|
||||||
@@ -55,6 +58,7 @@ from app.routes import ( # noqa: E402 — logging is configured first
|
|||||||
setup as setup_routes,
|
setup as setup_routes,
|
||||||
sync,
|
sync,
|
||||||
tags,
|
tags,
|
||||||
|
tokens as tokens_routes,
|
||||||
version,
|
version,
|
||||||
)
|
)
|
||||||
from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 — logging is configured first
|
from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 — logging is configured first
|
||||||
@@ -63,6 +67,10 @@ from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 —
|
|||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: FastAPI):
|
async def lifespan(app: FastAPI):
|
||||||
log.info("Siftlode starting up")
|
log.info("Siftlode starting up")
|
||||||
|
problem = storage.files_root_problem(settings.files_root)
|
||||||
|
if problem:
|
||||||
|
# The rest of the app still serves; the files library cannot (the worker refuses to start).
|
||||||
|
log.critical("The files library is not reachable: %s. See docs/self-hosting.md.", problem)
|
||||||
# The stored copy of the log (log_events). The entrypoint has already migrated the schema.
|
# The stored copy of the log (log_events). The entrypoint has already migrated the schema.
|
||||||
logsink.start()
|
logsink.start()
|
||||||
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
||||||
@@ -78,6 +86,18 @@ async def lifespan(app: FastAPI):
|
|||||||
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
||||||
# and `%s` of the exception does not say which of them raised.
|
# and `%s` of the exception does not say which of them raised.
|
||||||
log.exception("could not read the stored log level; staying at %s", current_level())
|
log.exception("could not read the stored log level; staying at %s", current_level())
|
||||||
|
if not problem:
|
||||||
|
try:
|
||||||
|
left = download_service.library_left_behind(db, settings.files_root)
|
||||||
|
except Exception: # a database not reachable yet: the check is a hint, never a gate
|
||||||
|
log.exception("could not check the files library against its database")
|
||||||
|
else:
|
||||||
|
if left:
|
||||||
|
log.critical(
|
||||||
|
"The files library is empty but the database names files: the volume "
|
||||||
|
"mounted at %s is not the one that holds them. See docs/self-hosting.md.",
|
||||||
|
settings.files_root,
|
||||||
|
)
|
||||||
if not state.is_configured(db):
|
if not state.is_configured(db):
|
||||||
token = state.rotate_setup_token(db)
|
token = state.rotate_setup_token(db)
|
||||||
url = f"{settings.app_base}/setup?token={token}"
|
url = f"{settings.app_base}/setup?token={token}"
|
||||||
@@ -267,7 +287,8 @@ async def static_cache_headers(request, call_next):
|
|||||||
# Baseline security response headers (SEC-1). A minimal, safe set that needs no per-route tuning:
|
# Baseline security response headers (SEC-1). A minimal, safe set that needs no per-route tuning:
|
||||||
# nosniff stops MIME-confusion, the Referrer-Policy keeps the (private) app URL from leaking to
|
# nosniff stops MIME-confusion, the Referrer-Policy keeps the (private) app URL from leaking to
|
||||||
# external sites the user follows, `frame-ancestors 'none'` blocks click-jacking by refusing to be
|
# external sites the user follows, `frame-ancestors 'none'` blocks click-jacking by refusing to be
|
||||||
# iframed anywhere (we never embed our own pages; we embed OTHERS, which this does not affect), and
|
# iframed anywhere (we embed OTHERS, which this does not affect; the one own answer the app frames, a
|
||||||
|
# library file's preview, sets its own `frame-ancestors 'self'` — these are defaults, `setdefault`), and
|
||||||
# Permissions-Policy turns off powerful features the app never uses. `X-Frame-Options: DENY` duplicates
|
# Permissions-Policy turns off powerful features the app never uses. `X-Frame-Options: DENY` duplicates
|
||||||
# the frame-ancestors intent for legacy engines that never implemented CSP frame-ancestors (belt and
|
# the frame-ancestors intent for legacy engines that never implemented CSP frame-ancestors (belt and
|
||||||
# suspenders — modern browsers use the CSP directive and ignore the header). A full Content-Security-Policy
|
# suspenders — modern browsers use the CSP directive and ignore the header). A full Content-Security-Policy
|
||||||
@@ -316,6 +337,8 @@ app.include_router(saved_views.router)
|
|||||||
app.include_router(plex_routes.router)
|
app.include_router(plex_routes.router)
|
||||||
app.include_router(downloads.router)
|
app.include_router(downloads.router)
|
||||||
app.include_router(downloads.admin_router)
|
app.include_router(downloads.admin_router)
|
||||||
|
app.include_router(files_routes.router)
|
||||||
|
app.include_router(tokens_routes.router)
|
||||||
app.include_router(public_routes.router)
|
app.include_router(public_routes.router)
|
||||||
app.include_router(admin.router)
|
app.include_router(admin.router)
|
||||||
app.include_router(config_routes.router)
|
app.include_router(config_routes.router)
|
||||||
@@ -339,6 +362,8 @@ STATIC_DIR = Path(__file__).parent / "static_spa"
|
|||||||
# stay cached but forces revalidation (cheap 304 via the FileResponse ETag) on every load. The
|
# stay cached but forces revalidation (cheap 304 via the FileResponse ETag) on every load. The
|
||||||
# hashed bundles under /assets can cache forever (a new build changes their filename).
|
# hashed bundles under /assets can cache forever (a new build changes their filename).
|
||||||
INDEX_HTML = STATIC_DIR / "index.html"
|
INDEX_HTML = STATIC_DIR / "index.html"
|
||||||
|
# A public share link's own page: the player alone, without the app's bundle (frontend/watch.html).
|
||||||
|
WATCH_HTML = STATIC_DIR / "watch.html"
|
||||||
INDEX_HEADERS = {"Cache-Control": "no-cache"}
|
INDEX_HEADERS = {"Cache-Control": "no-cache"}
|
||||||
app.mount(
|
app.mount(
|
||||||
"/assets",
|
"/assets",
|
||||||
@@ -354,16 +379,16 @@ async def index() -> FileResponse:
|
|||||||
|
|
||||||
@app.get("/watch/{token}")
|
@app.get("/watch/{token}")
|
||||||
async def watch_page(token: str):
|
async def watch_page(token: str):
|
||||||
"""Serve the SPA for a public share link, but with per-video Open Graph tags injected so the
|
"""Serve the public player page for a share link, with per-video Open Graph tags injected so the
|
||||||
link unfurls richly (title/channel/thumbnail) in Messenger/social crawlers. A real browser
|
link unfurls richly (title/channel/thumbnail) in Messenger/social crawlers. A real browser
|
||||||
ignores the extra tags and hydrates WatchPage as usual; invalid/expired/password links fall
|
ignores the extra tags and boots WatchPage; invalid/expired/password links fall back to the
|
||||||
back to the generic card."""
|
generic card."""
|
||||||
from app.downloads import og
|
from app.downloads import og
|
||||||
|
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
rendered = og.render_watch_html(token, INDEX_HTML, db)
|
rendered = og.render_watch_html(token, WATCH_HTML, db)
|
||||||
if rendered is None:
|
if rendered is None:
|
||||||
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|
return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
|
||||||
return HTMLResponse(rendered, headers=INDEX_HEADERS)
|
return HTMLResponse(rendered, headers=INDEX_HEADERS)
|
||||||
|
|
||||||
|
|
||||||
@@ -381,4 +406,8 @@ async def spa_fallback(full_path: str) -> FileResponse:
|
|||||||
# Real SPA-root assets (welcome images, favicon, robots.txt) rarely change and have
|
# Real SPA-root assets (welcome images, favicon, robots.txt) rarely change and have
|
||||||
# stable names, so a moderate cache is safe and satisfies the cache-lifetime audit.
|
# stable names, so a moderate cache is safe and satisfies the cache-lifetime audit.
|
||||||
return FileResponse(candidate, headers={"Cache-Control": "public, max-age=604800"})
|
return FileResponse(candidate, headers={"Cache-Control": "public, max-age=604800"})
|
||||||
|
# A /watch/ path the route above does not match (a trailing segment, no token) is still the
|
||||||
|
# player's: WatchPage reads the token off the path and says when a link is gone.
|
||||||
|
if full_path.startswith("watch/"):
|
||||||
|
return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
|
||||||
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|
||||||
|
|||||||
+182
-25
@@ -1,4 +1,4 @@
|
|||||||
from datetime import date, datetime
|
from datetime import date, datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import (
|
from sqlalchemy import (
|
||||||
BigInteger,
|
BigInteger,
|
||||||
@@ -18,6 +18,7 @@ from sqlalchemy import (
|
|||||||
Text,
|
Text,
|
||||||
UniqueConstraint,
|
UniqueConstraint,
|
||||||
func,
|
func,
|
||||||
|
text,
|
||||||
)
|
)
|
||||||
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||||
@@ -38,7 +39,7 @@ def display_text(raw: str | None) -> str | None:
|
|||||||
blank -> None.
|
blank -> None.
|
||||||
|
|
||||||
One definition because there are SIX writers and they had drifted: `POST /api/downloads`,
|
One definition because there are SIX writers and they had drifted: `POST /api/downloads`,
|
||||||
`POST /api/downloads/edit`, `POST /api/downloads/upload`, the metadata PATCH's two fields, and
|
`POST /api/downloads/edit`, `POST /api/files/upload`, the library PATCH's two fields, and
|
||||||
the Google sign-in callback. Only the PATCH truncated. Truncating is the right answer rather
|
the Google sign-in callback. Only the PATCH truncated. Truncating is the right answer rather
|
||||||
than refusing: the name is a label on work the user has already done — a file already
|
than refusing: the name is a label on work the user has already done — a file already
|
||||||
transferred, an account already authenticated — and losing its tail is not worth failing the
|
transferred, an account already authenticated — and losing its tail is not worth failing the
|
||||||
@@ -123,7 +124,8 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
|||||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
is_active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||||
# Admin-imposed access block, distinct from is_active (approval lifecycle). A suspended
|
# Admin-imposed access block, distinct from is_active (approval lifecycle). A suspended
|
||||||
# account can't sign in by any method and its live sessions are rejected; the user is told
|
# account can't sign in by any method and its live sessions are rejected; the user is told
|
||||||
# why and pointed at the operator's contact.
|
# why and pointed at the operator's contact. The worker leaves its queued downloads unclaimed
|
||||||
|
# until the block is lifted (`worker._claim_job`).
|
||||||
is_suspended: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
is_suspended: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
# Server-side session revocation (SA4): a monotonic counter bumped on password reset, password
|
# Server-side session revocation (SA4): a monotonic counter bumped on password reset, password
|
||||||
# change, or "log out everywhere". A signed session cookie records the epoch it was minted under;
|
# change, or "log out everywhere". A signed session cookie records the epoch it was minted under;
|
||||||
@@ -462,6 +464,27 @@ class VideoState(Base, UpdatedAtMixin):
|
|||||||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
|
||||||
|
class LibraryState(Base):
|
||||||
|
"""Where a user stopped in a library item (their own download, or one shared with them) — the
|
||||||
|
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
|
||||||
|
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
|
||||||
|
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
|
||||||
|
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
|
||||||
|
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
|
||||||
|
|
||||||
|
__tablename__ = "library_states"
|
||||||
|
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
|
job_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||||
|
)
|
||||||
|
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||||
|
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
class SearchFind(Base, TimestampMixin):
|
class SearchFind(Base, TimestampMixin):
|
||||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||||
|
|
||||||
@@ -804,12 +827,13 @@ class MediaAsset(Base, TimestampMixin):
|
|||||||
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
||||||
shared; per-user edited derivatives (phase 2) are never cached here.
|
shared; per-user edited derivatives (phase 2) are never cached here.
|
||||||
|
|
||||||
`rel_path` is relative to `settings.download_root` (the mount), so the physical filename is
|
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
|
||||||
system-decided and bound to the source id — never renamed (the user's custom name is display
|
system-decided and bound to the source id — never renamed (the user's custom name is display
|
||||||
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
||||||
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
||||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here)
|
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
|
||||||
plus `last_access_at`/`expires_at` drive retention + LRU eviction in the GC job."""
|
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
|
||||||
|
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
|
||||||
|
|
||||||
__tablename__ = "media_assets"
|
__tablename__ = "media_assets"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
@@ -825,10 +849,10 @@ class MediaAsset(Base, TimestampMixin):
|
|||||||
status: Mapped[str] = mapped_column(
|
status: Mapped[str] = mapped_column(
|
||||||
String(16), default="pending", server_default="pending", index=True
|
String(16), default="pending", server_default="pending", index=True
|
||||||
) # "pending" | "ready" | "error"
|
) # "pending" | "ready" | "error"
|
||||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root
|
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
|
||||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||||
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
||||||
# card / watch page / link preview can still show an image. Relative to download_root.
|
# card / watch page / link preview can still show an image. Relative to files_root.
|
||||||
poster_path: Mapped[str | None] = mapped_column(Text)
|
poster_path: Mapped[str | None] = mapped_column(Text)
|
||||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||||
@@ -844,8 +868,6 @@ class MediaAsset(Base, TimestampMixin):
|
|||||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
# Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
|
|
||||||
gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
||||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||||
title: Mapped[str | None] = mapped_column(Text)
|
title: Mapped[str | None] = mapped_column(Text)
|
||||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||||
@@ -855,13 +877,21 @@ class MediaAsset(Base, TimestampMixin):
|
|||||||
upload_date: Mapped[date | None] = mapped_column(Date)
|
upload_date: Mapped[date | None] = mapped_column(Date)
|
||||||
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
||||||
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
||||||
# reference shown on the Downloads page. Null until the worker extracts it (older/queued rows
|
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
|
||||||
# fall back to a URL derived from source_kind+source_ref).
|
# fall back to a URL derived from source_kind+source_ref).
|
||||||
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
||||||
error: Mapped[str | None] = mapped_column(Text)
|
error: Mapped[str | None] = mapped_column(Text)
|
||||||
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
# The file's SHA-256 (hex), so a machine fetching it can check what it got. Hashed while an upload
|
||||||
|
# streams in, otherwise on the first ask (`routes.files`); NULL = not hashed yet.
|
||||||
|
sha256: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
|
||||||
|
@validates("rel_path")
|
||||||
|
def _forget_hash(self, _key: str, rel_path: str | None) -> str | None:
|
||||||
|
# A hash names the bytes at `rel_path`: a new file there (a conversion, a retry) is not them.
|
||||||
|
self.sha256 = None
|
||||||
|
return rel_path
|
||||||
|
|
||||||
|
|
||||||
class DownloadProfile(Base, TimestampMixin):
|
class DownloadProfile(Base, TimestampMixin):
|
||||||
@@ -885,6 +915,92 @@ class DownloadProfile(Base, TimestampMixin):
|
|||||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
|
||||||
|
|
||||||
|
class FileFolder(Base, TimestampMixin):
|
||||||
|
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
|
||||||
|
|
||||||
|
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
|
||||||
|
user gets a file from that origin; an OWN folder (`name` set) is one the user made; the HUB folder
|
||||||
|
(neither set, one per user) is where they swap files with a machine of theirs. Every file —
|
||||||
|
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
|
||||||
|
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
|
||||||
|
|
||||||
|
__tablename__ = "file_folders"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||||
|
Index(
|
||||||
|
"uq_file_folder_own_name",
|
||||||
|
"user_id",
|
||||||
|
func.lower(text("name")),
|
||||||
|
unique=True,
|
||||||
|
postgresql_where=text("name IS NOT NULL"),
|
||||||
|
),
|
||||||
|
# One Hub folder per user.
|
||||||
|
Index("uq_file_folder_hub", "user_id", unique=True, postgresql_where=text("kind = 'hub'")),
|
||||||
|
CheckConstraint(
|
||||||
|
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||||
|
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)"
|
||||||
|
" OR (kind = 'hub' AND name IS NULL AND source_key IS NULL)",
|
||||||
|
name="ck_file_folder_kind",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
|
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
|
||||||
|
# "site:" + a host name (DNS caps one at 253 characters).
|
||||||
|
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||||
|
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||||
|
retention_days: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
|
||||||
|
|
||||||
|
class ApiToken(Base, TimestampMixin):
|
||||||
|
"""A user's token for a machine to reach their Files library with (`app.apitokens`): sent as
|
||||||
|
`Authorization: Bearer sfl_…`, honoured on `/api/files` only, and only in the folders its
|
||||||
|
grants name. Only the SHA-256 of the token is kept — it is shown once, when it is made — plus
|
||||||
|
its first characters (`prefix`), so the user can tell their tokens apart."""
|
||||||
|
|
||||||
|
__tablename__ = "api_tokens"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(80))
|
||||||
|
prefix: Mapped[str] = mapped_column(String(16))
|
||||||
|
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||||
|
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) # NULL = never
|
||||||
|
last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
# The one notice its user gets before it runs out (`apitokens.warn_expiring`).
|
||||||
|
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
|
||||||
|
|
||||||
|
class ApiTokenGrant(Base):
|
||||||
|
"""What one token may do in one folder. A folder with no grant is out of the token's reach."""
|
||||||
|
|
||||||
|
__tablename__ = "api_token_grants"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("token_id", "folder_id", name="uq_api_token_grant"),
|
||||||
|
CheckConstraint(
|
||||||
|
"can_list OR can_read OR can_write OR can_delete", name="ck_api_token_grant_some"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
token_id: Mapped[int] = mapped_column(ForeignKey("api_tokens.id", ondelete="CASCADE"))
|
||||||
|
folder_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
||||||
|
)
|
||||||
|
can_list: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
can_read: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
can_write: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
can_delete: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_on_insert(context) -> datetime | None:
|
||||||
|
"""A job inserted already done (an upload kept as it came) was ready the moment it was made; one
|
||||||
|
that becomes done later is stamped then (`folders.stamp_done`, `service.finalize_if_running`)."""
|
||||||
|
done = context.get_current_parameters().get("status") == "done"
|
||||||
|
return datetime.now(timezone.utc) if done else None
|
||||||
|
|
||||||
|
|
||||||
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||||
|
|
||||||
@@ -903,7 +1019,13 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
|
point at the parent download the clip was cut from; `edit_spec` is the (trim/crop) recipe."""
|
||||||
|
|
||||||
__tablename__ = "download_jobs"
|
__tablename__ = "download_jobs"
|
||||||
__table_args__ = (Index("ix_download_jobs_claim", "status", "queue_pos"),)
|
__table_args__ = (
|
||||||
|
Index("ix_download_jobs_claim", "status", "queue_pos"),
|
||||||
|
# Every way a job becomes done stamps when (`folders.stamp_done`, `finalize_if_running`).
|
||||||
|
CheckConstraint(
|
||||||
|
"status <> 'done' OR ready_at IS NOT NULL", name="ck_download_jobs_ready_at"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||||
@@ -912,7 +1034,9 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
)
|
)
|
||||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||||
source_ref: Mapped[str] = mapped_column(String(512))
|
source_ref: Mapped[str] = mapped_column(String(512))
|
||||||
# "download" (default) or "edit" (a per-user trim/crop derivative of another job).
|
# "download" (default), "edit" (a per-user trim/crop derivative of another job), "upload" (a
|
||||||
|
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`) or
|
||||||
|
# "clip" (a range cut straight from a YouTube video or a Plex item — see `downloads.clip`).
|
||||||
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
||||||
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
||||||
source_job_id: Mapped[int | None] = mapped_column(
|
source_job_id: Mapped[int | None] = mapped_column(
|
||||||
@@ -921,7 +1045,9 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
source_asset_id: Mapped[int | None] = mapped_column(
|
source_asset_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("media_assets.id", ondelete="SET NULL")
|
ForeignKey("media_assets.id", ondelete="SET NULL")
|
||||||
)
|
)
|
||||||
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}}.
|
# For edit jobs: the trim/crop recipe {trim?:{start_s,end_s}, crop?:{x,y,w,h}} in pixels. For clip
|
||||||
|
# jobs: the clip spec {source, ref, start_s, end_s, audio_track?, crop?:{x,y,w,h,aspect}}, its crop
|
||||||
|
# in fractions of the box it was drawn in (`downloads.clip.normalize`).
|
||||||
edit_spec: Mapped[dict | None] = mapped_column(JSON)
|
edit_spec: Mapped[dict | None] = mapped_column(JSON)
|
||||||
profile_id: Mapped[int | None] = mapped_column(
|
profile_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||||
@@ -948,6 +1074,27 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||||
|
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
|
||||||
|
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
|
||||||
|
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
|
||||||
|
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
|
||||||
|
folder_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
||||||
|
)
|
||||||
|
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
|
||||||
|
# traces its origin through its parent, and that parent may be deleted or expire later — the
|
||||||
|
# file must still know which source folder it may go back to. NULL only for a file placed before
|
||||||
|
# the column existed outside a source folder; `folders.source_key_for` works those out.
|
||||||
|
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||||
|
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||||
|
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
|
# When the file became done — what the library's "Added" means: a download that failed for days
|
||||||
|
# and then went through is new to its user, so it sorts as new. `created_at` stays when it was
|
||||||
|
# asked for (the queue's order). Set again when a file is made anew (a retry, a re-download).
|
||||||
|
ready_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), default=_ready_on_insert
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DownloadShare(Base, TimestampMixin):
|
class DownloadShare(Base, TimestampMixin):
|
||||||
@@ -982,21 +1129,33 @@ class DownloadQuota(Base, UpdatedAtMixin):
|
|||||||
|
|
||||||
|
|
||||||
class DownloadLink(Base, TimestampMixin):
|
class DownloadLink(Base, TimestampMixin):
|
||||||
"""A public capability link to one download — the "share by link" surface.
|
"""A public capability link to one download, or to one of its owner's own folders — the "share
|
||||||
|
by link" surface.
|
||||||
|
|
||||||
Anyone holding the unguessable `token` can watch the file on the login-free `/watch/{token}`
|
Anyone holding the unguessable token can watch the file on the login-free `/watch/{token}`
|
||||||
player page (Google-Drive style). Optional per-link controls: `expires_at`, `allow_download`
|
player page (Google-Drive style); a folder link opens the folder's list there, with what is in
|
||||||
(stream-only vs downloadable), and a `password_hash` (argon2, like a user password). Revoke =
|
the folder at the time it is opened. The token itself is never stored as written: a link is
|
||||||
delete the row. Distinct from `DownloadShare`, which grants a *registered* user access via the
|
found by its SHA-256 (`token_hash`), and `token_enc` is the Fernet-encrypted copy its owner
|
||||||
in-app "Shared with me" list."""
|
copies the link from again — absent without a key (`downloads/links.py`). Optional per-link
|
||||||
|
controls: `expires_at`, `allow_download` (stream-only vs downloadable), and a `password_hash`
|
||||||
|
(argon2, like a user password). Revoke = delete the row. Distinct from `DownloadShare`, which
|
||||||
|
grants a *registered* user access via the in-app "Shared with me" list."""
|
||||||
|
|
||||||
__tablename__ = "download_links"
|
__tablename__ = "download_links"
|
||||||
|
__table_args__ = (
|
||||||
|
# A link opens exactly one thing: a file or a folder.
|
||||||
|
CheckConstraint("(job_id IS NULL) <> (folder_id IS NULL)", name="ck_download_link_target"),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True)
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
job_id: Mapped[int] = mapped_column(
|
job_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||||
)
|
)
|
||||||
token: Mapped[str] = mapped_column(String(64), unique=True)
|
folder_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("file_folders.id", ondelete="CASCADE"), index=True
|
||||||
|
)
|
||||||
|
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||||
|
token_enc: Mapped[str | None] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||||
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
created_by: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
allow_download: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||||
password_hash: Mapped[str | None] = mapped_column(Text)
|
password_hash: Mapped[str | None] = mapped_column(Text)
|
||||||
@@ -1252,8 +1411,6 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
# bit-depth/HDR. NULL = never (the backfill's work-list). Only HW-decodable transcode items are
|
# bit-depth/HDR. NULL = never (the backfill's work-list). Only HW-decodable transcode items are
|
||||||
# probed (the rest are SW-only by codec and need no bit-depth); see `_probe_media_batch`.
|
# probed (the rest are SW-only by codec and need no bit-depth); see `_probe_media_batch`.
|
||||||
media_probed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
media_probed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
# Intro/credit markers: [{"type": "intro"|"credits", "start_s": int, "end_s": int}, ...]
|
|
||||||
markers: Mapped[list | None] = mapped_column(JSON)
|
|
||||||
added_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
added_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||||
# Filterable / orderable metadata mirrored from the Plex section listing (no extra API calls;
|
# Filterable / orderable metadata mirrored from the Plex section listing (no extra API calls;
|
||||||
# `rating` = Plex audienceRating, which is ~the IMDb score). The arrays back genre/person/studio
|
# `rating` = Plex audienceRating, which is ~the IMDb score). The arrays back genre/person/studio
|
||||||
|
|||||||
@@ -10,4 +10,4 @@ from pathlib import Path
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
CACHE_DIR = Path(settings.download_root) / ".plex-img-cache"
|
CACHE_DIR = Path(settings.files_root) / ".plex-img-cache"
|
||||||
|
|||||||
@@ -3,9 +3,24 @@ playlists, and detail views render. Extracted from routes/plex/browse.py so the
|
|||||||
lives beside the other Plex read-model modules (detail.py, query.py) rather than inside the
|
lives beside the other Plex read-model modules (detail.py, query.py) rather than inside the
|
||||||
router. Pure functions — no DB, no request state."""
|
router. Pure functions — no DB, no request state."""
|
||||||
|
|
||||||
|
from typing import NamedTuple
|
||||||
|
|
||||||
from app.models import PlexItem, PlexShow, PlexState
|
from app.models import PlexItem, PlexShow, PlexState
|
||||||
|
|
||||||
|
|
||||||
|
class ShowState(NamedTuple):
|
||||||
|
"""A show's watch state rolled up from its episodes, as its card carries it. `resumable`: an unwatched
|
||||||
|
episode has a resume point, the one the card's Continue resumes and its Restart replays from 0. A
|
||||||
|
show part watched with its next episode untouched has none: Continue and Restart would both play
|
||||||
|
that episode from its start, so the card offers Play alone."""
|
||||||
|
|
||||||
|
status: str
|
||||||
|
resumable: bool
|
||||||
|
|
||||||
|
|
||||||
|
SHOW_NEW = ShowState("new", False)
|
||||||
|
|
||||||
|
|
||||||
# A card's wide backdrop, for a rail's landscape tile or a crossfading hero — but ONLY when the item
|
# A card's wide backdrop, for a rail's landscape tile or a crossfading hero — but ONLY when the item
|
||||||
# actually has art. `thumb` is always emitted (its endpoint placeholders a missing poster), but an
|
# actually has art. `thumb` is always emitted (its endpoint placeholders a missing poster), but an
|
||||||
# `art` URL for an item with no `art_key` would resolve to a broken/blank image on every card that
|
# `art` URL for an item with no `art_key` would resolve to a broken/blank image on every card that
|
||||||
@@ -54,7 +69,7 @@ def movie_card(it: PlexItem, st: PlexState | None, *, lang: str) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def show_card(sh: PlexShow, status: str = "new", *, lang: str) -> dict:
|
def show_card(sh: PlexShow, state: ShowState = SHOW_NEW, *, lang: str) -> dict:
|
||||||
return {
|
return {
|
||||||
"id": sh.rating_key,
|
"id": sh.rating_key,
|
||||||
"type": "show",
|
"type": "show",
|
||||||
@@ -66,7 +81,8 @@ def show_card(sh: PlexShow, status: str = "new", *, lang: str) -> dict:
|
|||||||
"art": _art_url(sh.rating_key, sh.art_key),
|
"art": _art_url(sh.rating_key, sh.art_key),
|
||||||
"season_count": sh.child_count,
|
"season_count": sh.child_count,
|
||||||
# Aggregate watch-state across the show's episodes (new | in_progress | watched) → grid badge.
|
# Aggregate watch-state across the show's episodes (new | in_progress | watched) → grid badge.
|
||||||
"status": status,
|
"status": state.status,
|
||||||
|
"resumable": state.resumable,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -167,12 +167,16 @@ class PlexClient:
|
|||||||
keys = ",".join(str(k) for k in rating_keys)
|
keys = ",".join(str(k) for k in rating_keys)
|
||||||
return f"server://{self.machine_id}/com.plexapp.plugins.library/library/metadata/{keys}"
|
return f"server://{self.machine_id}/com.plexapp.plugins.library/library/metadata/{keys}"
|
||||||
|
|
||||||
def create_collection(self, section_key: str, title: str, item_rating_key: str) -> dict:
|
def create_collection(
|
||||||
"""Create a (dumb) collection seeded with one item; returns its Metadata (incl. ratingKey)."""
|
self, section_key: str, title: str, item_rating_key: str, item_type: int = 1
|
||||||
|
) -> dict:
|
||||||
|
"""Create a (dumb) collection seeded with one item; returns its Metadata (incl. ratingKey).
|
||||||
|
`item_type` is the Plex metadata type of its members — 1 movie, 2 show. Measured: a collection
|
||||||
|
made as type 1 in a TV section came out EMPTY with a show seed, and refused the show after."""
|
||||||
uri = quote(self._metadata_uri(item_rating_key), safe="")
|
uri = quote(self._metadata_uri(item_rating_key), safe="")
|
||||||
mc = self._write(
|
mc = self._write(
|
||||||
"POST",
|
"POST",
|
||||||
f"/library/collections?type=1&smart=0§ionId={section_key}"
|
f"/library/collections?type={item_type}&smart=0§ionId={section_key}"
|
||||||
f"&title={quote(title, safe='')}&uri={uri}",
|
f"&title={quote(title, safe='')}&uri={uri}",
|
||||||
)
|
)
|
||||||
items = mc.get("Metadata", []) or []
|
items = mc.get("Metadata", []) or []
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ the two never diverge — the same one-parser rule that keeps `people.py`'s cast
|
|||||||
(genres/directors/studio/content_rating already land from the cheap listing at `_apply_facet_fields`)."""
|
(genres/directors/studio/content_rating already land from the cheap listing at `_apply_facet_fields`)."""
|
||||||
|
|
||||||
from app.plex import people
|
from app.plex import people
|
||||||
|
from app.plex.plaintext import plain_text
|
||||||
|
|
||||||
|
|
||||||
def rich_detail(meta: dict) -> dict:
|
def rich_detail(meta: dict) -> dict:
|
||||||
@@ -42,7 +43,7 @@ def rich_detail(meta: dict) -> dict:
|
|||||||
"genres": [g.get("tag") for g in (meta.get("Genre") or []) if g.get("tag")][:8],
|
"genres": [g.get("tag") for g in (meta.get("Genre") or []) if g.get("tag")][:8],
|
||||||
"directors": [d.get("tag") for d in (meta.get("Director") or []) if d.get("tag")][:4],
|
"directors": [d.get("tag") for d in (meta.get("Director") or []) if d.get("tag")][:4],
|
||||||
"studio": meta.get("studio"),
|
"studio": meta.get("studio"),
|
||||||
"tagline": meta.get("tagline"),
|
"tagline": plain_text(meta.get("tagline")),
|
||||||
"cast": people.cast_from_meta(meta),
|
"cast": people.cast_from_meta(meta),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from app.models import PlexItem, PlexLibrary, PlexShow
|
|||||||
from app.plex import sync as plex_sync
|
from app.plex import sync as plex_sync
|
||||||
from app.plex.client import PlexClient
|
from app.plex.client import PlexClient
|
||||||
from app.plex.genres import hu_genres
|
from app.plex.genres import hu_genres
|
||||||
|
from app.plex.plaintext import plain_text
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
@@ -109,8 +110,8 @@ def _overlay_pass(
|
|||||||
for g, meta in hu_by_guid.items():
|
for g, meta in hu_by_guid.items():
|
||||||
targets = by_guid[g]
|
targets = by_guid[g]
|
||||||
title_hu = (meta.get("title") or "").strip() or None
|
title_hu = (meta.get("title") or "").strip() or None
|
||||||
summary_hu = meta.get("summary")
|
summary_hu = plain_text(meta.get("summary"))
|
||||||
tagline_hu = (meta.get("tagline") or "").strip() or None
|
tagline_hu = plain_text(meta.get("tagline"))
|
||||||
genres_hu = hu_genres(plex_sync._tags(meta, "Genre"))
|
genres_hu = hu_genres(plex_sync._tags(meta, "Genre"))
|
||||||
for row in targets:
|
for row in targets:
|
||||||
row.title_hu = title_hu
|
row.title_hu = title_hu
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
"""Plex text fields as plain text.
|
||||||
|
|
||||||
|
Plex hands a summary over as the agent wrote it, and some agents (a Hungarian TV listing, for one)
|
||||||
|
write HTML: `<p>…</p>` paragraphs and `á`-style entities. The app renders summaries as text,
|
||||||
|
so the tags showed literally on the info page and the hover card, and a card's 300-character cut
|
||||||
|
could land inside one. Every summary and tagline is stored through `plain_text`, and the migration
|
||||||
|
that cleaned the existing rows calls it too, so the two cannot disagree.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import html
|
||||||
|
import re
|
||||||
|
from html.entities import html5
|
||||||
|
|
||||||
|
_BREAK = re.compile(r"<br\s*/?>", re.IGNORECASE)
|
||||||
|
# A paragraph or block boundary: a blank line between the two. Only a tag name ends the match, so a
|
||||||
|
# bare "<" in prose ("a < b", "<3") is left alone.
|
||||||
|
_BLOCK = re.compile(r"</?(?:p|div|li|ul|ol|h[1-6]|blockquote)\b[^>]*>", re.IGNORECASE)
|
||||||
|
_TAG = re.compile(r"</?[a-zA-Z][^>]*>")
|
||||||
|
_BLANKS = re.compile(r"[ \t]*\n[ \t]*\n\s*")
|
||||||
|
|
||||||
|
# A well-formed character reference only. `html.unescape` alone also decodes the legacy forms that
|
||||||
|
# have no semicolon, so prose like "AT¬ice" or "R©" came out as "AT¬ice" and "R©". A named
|
||||||
|
# one counts only when its whole name is an entity: for "¬ice;" `html.unescape` still falls back
|
||||||
|
# to the legacy "¬" inside it.
|
||||||
|
_REF = re.compile(r"&(?:#[0-9]+|#[xX][0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]*);")
|
||||||
|
|
||||||
|
|
||||||
|
def _decode(m: re.Match[str]) -> str:
|
||||||
|
ref = m.group()
|
||||||
|
if ref[1] != "#":
|
||||||
|
return html.unescape(ref) if ref[1:] in html5 else ref
|
||||||
|
if ref[2] in "xX":
|
||||||
|
return html.unescape(ref)
|
||||||
|
# `int()` refuses a decimal run past 4300 digits, and any number past seven significant digits is
|
||||||
|
# beyond U+10FFFF, which the parser turns into U+FFFD anyway.
|
||||||
|
digits = ref[2:-1].lstrip("0") or "0"
|
||||||
|
return "�" if len(digits) > 7 else html.unescape(f"&#{digits};")
|
||||||
|
|
||||||
|
|
||||||
|
def plain_text(value: str | None) -> str | None:
|
||||||
|
"""`value` with its markup gone: line breaks kept as newlines, paragraphs as a blank line,
|
||||||
|
every other tag dropped and entities decoded. None or an all-blank result stays None."""
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if "<" in value or "&" in value:
|
||||||
|
value = _BREAK.sub("\n", value)
|
||||||
|
value = _BLOCK.sub("\n\n", value)
|
||||||
|
value = _TAG.sub("", value)
|
||||||
|
value = _REF.sub(_decode, value)
|
||||||
|
value = _BLANKS.sub("\n\n", value)
|
||||||
|
return value.strip() or None
|
||||||
@@ -9,6 +9,7 @@ from sqlalchemy import and_, case, func, or_, select
|
|||||||
from sqlalchemy.orm import Session, aliased
|
from sqlalchemy.orm import Session, aliased
|
||||||
|
|
||||||
from app.models import PlexItem, PlexState
|
from app.models import PlexItem, PlexState
|
||||||
|
from app.plex.cards import ShowState
|
||||||
|
|
||||||
# Sort sentinel for a rail row whose timestamp is missing, so `key=lambda r: r[0] or NO_TS` reads the
|
# Sort sentinel for a rail row whose timestamp is missing, so `key=lambda r: r[0] or NO_TS` reads the
|
||||||
# same at every rail that merges two type-homogeneous lists in Python.
|
# same at every rail that merges two type-homogeneous lists in Python.
|
||||||
@@ -32,6 +33,11 @@ def show_status(total: int, watched: int, inprog: int) -> str:
|
|||||||
return "new"
|
return "new"
|
||||||
|
|
||||||
|
|
||||||
|
def show_state(total: int, watched: int, inprog: int) -> ShowState:
|
||||||
|
"""`show_status` plus whether an episode holds a resume point (the card's `ShowState`)."""
|
||||||
|
return ShowState(show_status(total, watched, inprog), (inprog or 0) > 0)
|
||||||
|
|
||||||
|
|
||||||
def rollup(cards: list[dict]) -> dict:
|
def rollup(cards: list[dict]) -> dict:
|
||||||
"""From ORDERED episode cards → the aggregate watch status, the "on deck" episode to Resume (the
|
"""From ORDERED episode cards → the aggregate watch status, the "on deck" episode to Resume (the
|
||||||
last in-progress one, else the first unwatched), the first episode (for Play from the start), and
|
last in-progress one, else the first unwatched), the first episode (for Play from the start), and
|
||||||
@@ -92,16 +98,16 @@ def agg_subquery(db: Session, user_id: int, include_resumed: bool = False):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def state_map(db: Session, user_id: int, show_ids: list[int]) -> dict[int, str]:
|
def state_map(db: Session, user_id: int, show_ids: list[int]) -> dict[int, ShowState]:
|
||||||
if not show_ids:
|
if not show_ids:
|
||||||
return {}
|
return {}
|
||||||
rows = db.execute(agg_subquery(db, user_id).where(PlexItem.show_id.in_(show_ids))).all()
|
rows = db.execute(agg_subquery(db, user_id).where(PlexItem.show_id.in_(show_ids))).all()
|
||||||
return {r.show_id: show_status(r.total, r.watched, r.inprog) for r in rows}
|
return {r.show_id: show_state(r.total, r.watched, r.inprog) for r in rows}
|
||||||
|
|
||||||
|
|
||||||
def item_state_map(db: Session, user_id: int, item_ids: list[int]) -> dict[int, PlexState]:
|
def item_state_map(db: Session, user_id: int, item_ids: list[int]) -> dict[int, PlexState]:
|
||||||
"""A user's PlexState rows for a set of leaf items (movies or episodes), keyed by item_id — the
|
"""A user's PlexState rows for a set of leaf items (movies or episodes), keyed by item_id — the
|
||||||
leaf analogue of `state_map` (which rolls a show's episodes up to a status string). One batched
|
leaf analogue of `state_map` (which rolls a show's episodes up to a `ShowState`). One batched
|
||||||
fetch, so the caller serializes cards without a per-card query. Empty in → empty dict (no `IN ()`)."""
|
fetch, so the caller serializes cards without a per-card query. Empty in → empty dict (no `IN ()`)."""
|
||||||
if not item_ids:
|
if not item_ids:
|
||||||
return {}
|
return {}
|
||||||
|
|||||||
+31
-11
@@ -35,13 +35,13 @@ from app.config import settings
|
|||||||
from app.downloads.edit import encode_thread_cap
|
from app.downloads.edit import encode_thread_cap
|
||||||
from app.models import PlexItem
|
from app.models import PlexItem
|
||||||
from app.plex import paths
|
from app.plex import paths
|
||||||
from app.plex.sync import HW_DECODABLE_CODECS
|
from app.plex.sync import HDR_TRANSFERS, HW_DECODABLE_CODECS
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
_HLS_ROOT = (
|
_HLS_ROOT = (
|
||||||
Path(settings.plex_hls_dir)
|
Path(settings.plex_hls_dir)
|
||||||
if settings.plex_hls_dir
|
if settings.plex_hls_dir
|
||||||
else Path(settings.download_root) / ".plex-hls"
|
else Path(settings.files_root) / ".plex-hls"
|
||||||
)
|
)
|
||||||
_SEG_SECONDS = 6
|
_SEG_SECONDS = 6
|
||||||
# Cold-start warm-up. A freshly-spawned ffmpeg has produced only the first segment (or two) by the
|
# Cold-start warm-up. A freshly-spawned ffmpeg has produced only the first segment (or two) by the
|
||||||
@@ -203,7 +203,8 @@ _HW_DECODABLE = HW_DECODABLE_CODECS
|
|||||||
# plain scale_vaapi=format=nv12 path (Gen 9.5 VPP refuses it) — it needs the OpenCL tonemap graph
|
# plain scale_vaapi=format=nv12 path (Gen 9.5 VPP refuses it) — it needs the OpenCL tonemap graph
|
||||||
# (_venc_opts tonemap branch) when the OpenCL runtime is present, else software libx264. A 10-bit source
|
# (_venc_opts tonemap branch) when the OpenCL runtime is present, else software libx264. A 10-bit source
|
||||||
# WITHOUT one of these transfers is SDR, not HDR, and must not be tone-mapped. See _probe_video_kind.
|
# WITHOUT one of these transfers is SDR, not HDR, and must not be tone-mapped. See _probe_video_kind.
|
||||||
_HDR_TRANSFERS = {"smpte2084", "arib-std-b67"}
|
# THE single source is plex.sync (a clip's tonemap reads it too).
|
||||||
|
_HDR_TRANSFERS = HDR_TRANSFERS
|
||||||
|
|
||||||
|
|
||||||
# The three source classes that drive the transcode-pipeline choice (see _probe_video_kind).
|
# The three source classes that drive the transcode-pipeline choice (see _probe_video_kind).
|
||||||
@@ -744,8 +745,9 @@ def start_session(
|
|||||||
# `replace_only` makes the supersede identity-aware for the runtime fallback: the whole
|
# `replace_only` makes the supersede identity-aware for the runtime fallback: the whole
|
||||||
# check-and-replace happens under ONE _lock, so a newer same-key session (a concurrent
|
# check-and-replace happens under ONE _lock, so a newer same-key session (a concurrent
|
||||||
# seek-restart) installed between the fallback's decision and here is never clobbered. When the
|
# seek-restart) installed between the fallback's decision and here is never clobbered. When the
|
||||||
# registered session is no longer `replace_only`, we abort and return None (the caller yields to
|
# registered session is no longer `replace_only`, we abort and return None, and the caller
|
||||||
# whoever is registered now). The INITIAL spawn never gets None — see its call site.
|
# answers with its own dead session — never the newer one, whose id a stale requester would
|
||||||
|
# stop. The INITIAL spawn never gets None — see its call site.
|
||||||
# Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and
|
# Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and
|
||||||
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
||||||
preset, crf, max_height = "veryfast", 20, 0
|
preset, crf, max_height = "veryfast", 20, 0
|
||||||
@@ -840,12 +842,13 @@ def start_session(
|
|||||||
)
|
)
|
||||||
respawned = _spawn(False, False, replace_only=s)
|
respawned = _spawn(False, False, replace_only=s)
|
||||||
if respawned is None:
|
if respawned is None:
|
||||||
# Superseded/culled mid-fallback — yield whoever is registered now (a newer seek-restart), or
|
# Superseded/culled mid-fallback — answer with our own now-dead session. Never the newer one
|
||||||
# our own now-dead session if the key was cap-culled. Never None here: the POST /session route
|
# registered meanwhile: this answer goes to the request that LOST, whose player drops it as
|
||||||
# turns a None return into a "Media file not found on disk" 404 — dishonest for a file that
|
# stale and stops the session it names (`stop_session`), which stopped the playback that won.
|
||||||
# exists. A (dead) session instead degrades to a retryable 404 ("No active playback session" /
|
# Never None either: the POST /session route turns a None return into a "Media file not found
|
||||||
# "Not available yet") the client re-requests.
|
# on disk" 404 — dishonest for a file that exists. A dead session instead degrades to a
|
||||||
return current_session(user_id, key) or s
|
# retryable 404 ("No active playback session" / "Not available yet") the client re-requests.
|
||||||
|
return s
|
||||||
s = respawned
|
s = respawned
|
||||||
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
||||||
if ready:
|
if ready:
|
||||||
@@ -1117,6 +1120,23 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int:
|
|||||||
return dropped
|
return dropped
|
||||||
|
|
||||||
|
|
||||||
|
def stop_session(user_id: int, key: str, session_id: str) -> bool:
|
||||||
|
"""Stop this viewer's session for `key` now, because the player that asked for it is gone (left
|
||||||
|
the item, closed, unloaded the page). Left to `reap_idle`, a transcode ran on to the end of the
|
||||||
|
film first — measured: ~4 cores for a minute after leaving a resumed 480p film.
|
||||||
|
|
||||||
|
Only while `session_id` is still the registered one: the player names the session it played, and
|
||||||
|
a seek-restart or a second tab may have replaced it meanwhile — that newer session is somebody's
|
||||||
|
current playback, not this request's to end."""
|
||||||
|
with _lock:
|
||||||
|
s = _sessions.get((user_id, key))
|
||||||
|
if s is None or s.id != session_id:
|
||||||
|
return False
|
||||||
|
_kill(s)
|
||||||
|
_sessions.pop((user_id, key), None)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def touch_session(user_id: int, key: str) -> None:
|
def touch_session(user_id: int, key: str) -> None:
|
||||||
"""Say this viewer is still there for `key`, without fetching anything. `last_access` used to move
|
"""Say this viewer is still there for `key`, without fetching anything. `last_access` used to move
|
||||||
only on an HLS file fetch, which a paused player does not do — so the only way to keep a paused
|
only on an HLS file fetch, which a paused player does not do — so the only way to keep a paused
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ from app.plex import detail, overlay, people
|
|||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
from app.plex.genres import hu_genres
|
from app.plex.genres import hu_genres
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
|
from app.plex.plaintext import plain_text
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
||||||
@@ -63,9 +64,11 @@ def classify_playable(vcodec: str | None, acodec: str | None, container: str | N
|
|||||||
# outside it (AV1, MPEG-4 ASP/XviD…) is software-only even on the iGPU (it can't decode it to feed its
|
# outside it (AV1, MPEG-4 ASP/XviD…) is software-only even on the iGPU (it can't decode it to feed its
|
||||||
# own encoder).
|
# own encoder).
|
||||||
HW_DECODABLE_CODECS = frozenset({"h264", "hevc", "h265", "vp9", "vc1", "mpeg2video"})
|
HW_DECODABLE_CODECS = frozenset({"h264", "hevc", "h265", "vp9", "vc1", "mpeg2video"})
|
||||||
# Color-transfer values marking an HDR source (PQ / HLG). A 10-bit/HDR source breaks the VAAPI
|
# Color-transfer values marking an HDR source (PQ / HLG). THE single source of this set: plex.stream
|
||||||
# scale=nv12 path on Gen 9.5 (see plex.stream) → software transcode, so it counts as NOT HW-capable.
|
# (the tonemap choice) and downloads.clip (a clip's tonemap) import it here, so a transfer added for
|
||||||
_HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"})
|
# one is tonemapped by all. A 10-bit/HDR source breaks the VAAPI scale=nv12 path on Gen 9.5 (see
|
||||||
|
# plex.stream) → software transcode, so it counts as NOT HW-capable.
|
||||||
|
HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"})
|
||||||
|
|
||||||
|
|
||||||
def transcode_hw_capable(
|
def transcode_hw_capable(
|
||||||
@@ -106,7 +109,7 @@ def _video_stream_props(meta: dict) -> tuple[int | None, bool]:
|
|||||||
# whole library would drown the sync's own lines.
|
# whole library would drown the sync's own lines.
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
depth = None
|
depth = None
|
||||||
hdr = str(s.get("colorTrc") or "").lower() in _HDR_TRANSFERS
|
hdr = str(s.get("colorTrc") or "").lower() in HDR_TRANSFERS
|
||||||
return depth, hdr
|
return depth, hdr
|
||||||
return None, False
|
return None, False
|
||||||
|
|
||||||
@@ -468,7 +471,7 @@ def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
|
|||||||
# carries none (a server ignoring includeGuids mustn't wipe the overlay's only match key).
|
# carries none (a server ignoring includeGuids mustn't wipe the overlay's only match key).
|
||||||
row.guid = _canonical_guid(meta) or row.guid
|
row.guid = _canonical_guid(meta) or row.guid
|
||||||
row.title = (meta.get("title") or "").strip() or "Untitled"
|
row.title = (meta.get("title") or "").strip() or "Untitled"
|
||||||
row.summary = meta.get("summary")
|
row.summary = plain_text(meta.get("summary"))
|
||||||
row.year = meta.get("year")
|
row.year = meta.get("year")
|
||||||
row.duration_s = _dur_s(meta)
|
row.duration_s = _dur_s(meta)
|
||||||
row.thumb_key = meta.get("thumb")
|
row.thumb_key = meta.get("thumb")
|
||||||
@@ -576,7 +579,7 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
|||||||
_canonical_guid(meta) or sh.guid
|
_canonical_guid(meta) or sh.guid
|
||||||
) # EN<->HU pairing key; keep prior if listing has none
|
) # EN<->HU pairing key; keep prior if listing has none
|
||||||
sh.title = (meta.get("title") or "").strip() or "Untitled"
|
sh.title = (meta.get("title") or "").strip() or "Untitled"
|
||||||
sh.summary = meta.get("summary")
|
sh.summary = plain_text(meta.get("summary"))
|
||||||
sh.year = meta.get("year")
|
sh.year = meta.get("year")
|
||||||
sh.thumb_key = meta.get("thumb")
|
sh.thumb_key = meta.get("thumb")
|
||||||
sh.art_key = meta.get("art")
|
sh.art_key = meta.get("art")
|
||||||
@@ -659,7 +662,7 @@ def _upsert_collection(
|
|||||||
db.add(col)
|
db.add(col)
|
||||||
col.library_id = lib_id
|
col.library_id = lib_id
|
||||||
col.title = (meta.get("title") or "").strip() or "Untitled"
|
col.title = (meta.get("title") or "").strip() or "Untitled"
|
||||||
col.summary = meta.get("summary")
|
col.summary = plain_text(meta.get("summary"))
|
||||||
col.thumb_key = meta.get("thumb")
|
col.thumb_key = meta.get("thumb")
|
||||||
col.child_count = meta.get("childCount")
|
col.child_count = meta.get("childCount")
|
||||||
col.smart = str(meta.get("smart")) == "1"
|
col.smart = str(meta.get("smart")) == "1"
|
||||||
|
|||||||
@@ -6,19 +6,55 @@ and "fetch a row this user owns or 404" idioms have one definition, one status c
|
|||||||
one existence-leak policy.
|
one existence-leak policy.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import IO, Any, TypeVar
|
from typing import Any, Protocol, TypeVar
|
||||||
|
|
||||||
from fastapi import HTTPException, Request
|
from fastapi import HTTPException, Request
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.db import Base
|
from app.db import Base
|
||||||
from app.models import User
|
from app.models import POSITION_MAX_S, User
|
||||||
from app.issues import gitea
|
from app.issues import gitea
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
from app.youtube.client import YouTubeError, refusal_reason
|
from app.youtube.client import YouTubeError, refusal_reason
|
||||||
|
|
||||||
M = TypeVar("M", bound=Base)
|
M = TypeVar("M", bound=Base)
|
||||||
|
|
||||||
|
# Resume-position thresholds of a player that only keeps a resume point (the YouTube player, the
|
||||||
|
# library's): positions below this are "didn't really start" and within this of the end are
|
||||||
|
# "basically finished" — neither is worth storing, so the position is cleared instead (no progress
|
||||||
|
# bar). The client's local resume (`lib/resume.ts`) keeps the same two numbers, held equal by
|
||||||
|
# `resume.contract.test.ts` beside it. NB: the Plex player intentionally uses a LARGER finish margin
|
||||||
|
# (30s) because it marks long-form items watched, not just clears a resume bar — see
|
||||||
|
# routes/plex/playback.py.
|
||||||
|
PROGRESS_MIN_SECONDS = 5
|
||||||
|
FINISH_MARGIN_SECONDS = 10
|
||||||
|
|
||||||
|
|
||||||
|
class ProgressIn(BaseModel):
|
||||||
|
"""A player's resume checkpoint (sent every few seconds while it plays, and as a beacon when the
|
||||||
|
page goes away)."""
|
||||||
|
|
||||||
|
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||||
|
# one into a 500.
|
||||||
|
model_config = ConfigDict(allow_inf_nan=False)
|
||||||
|
|
||||||
|
# Accept a float and truncate (`kept_position`) — the player may post a raw currentTime (a float),
|
||||||
|
# which the old `int(payload.get(...))` silently truncated. A plain `int` field would 422 that on
|
||||||
|
# the progress hot path, losing the resume position; `float` keeps the old tolerance.
|
||||||
|
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
||||||
|
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||||
|
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||||
|
|
||||||
|
|
||||||
|
def kept_position(body: ProgressIn) -> int:
|
||||||
|
"""The whole-second resume point worth storing for `body`, or 0 for "clear it" (too early to
|
||||||
|
count as started, or close enough to the end to count as finished)."""
|
||||||
|
position = max(0, int(body.position_seconds))
|
||||||
|
duration = int(body.duration_seconds)
|
||||||
|
near_end = duration > 0 and position > duration - FINISH_MARGIN_SECONDS
|
||||||
|
return position if position >= PROGRESS_MIN_SECONDS and not near_end else 0
|
||||||
|
|
||||||
|
|
||||||
def not_found(what: str = "generic") -> HTTPException:
|
def not_found(what: str = "generic") -> HTTPException:
|
||||||
"""The 404 for a row that is gone — structured, so the bilingual UI says it in the reader's
|
"""The 404 for a row that is gone — structured, so the bilingual UI says it in the reader's
|
||||||
@@ -86,8 +122,14 @@ async def capped_body(request: Request, limit: int) -> bytes:
|
|||||||
return bytes(body)
|
return bytes(body)
|
||||||
|
|
||||||
|
|
||||||
|
class ByteSink(Protocol):
|
||||||
|
"""Where `capped_stream_to` writes: a file, or anything that takes the bytes on their way to one."""
|
||||||
|
|
||||||
|
def write(self, data: bytes, /) -> object: ...
|
||||||
|
|
||||||
|
|
||||||
async def capped_stream_to(
|
async def capped_stream_to(
|
||||||
request: Request, out: IO[bytes], limit: int, refusal: HTTPException
|
request: Request, out: ByteSink, limit: int, refusal: HTTPException
|
||||||
) -> int:
|
) -> int:
|
||||||
"""`capped_body`'s sibling for a body too large to hold: the same counting rule, written
|
"""`capped_body`'s sibling for a body too large to hold: the same counting rule, written
|
||||||
straight to `out` instead of accumulated, returning the size.
|
straight to `out` instead of accumulated, returning the size.
|
||||||
|
|||||||
@@ -484,7 +484,8 @@ def set_user_suspended(
|
|||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Suspend or un-suspend an account. A suspended user can't sign in by any method and any
|
"""Suspend or un-suspend an account. A suspended user can't sign in by any method and any
|
||||||
live session is dropped on its next request. Guards: can't suspend yourself, and can't suspend the last active admin (that would lock admin out). On
|
live session is dropped on its next request; their queued downloads wait (the worker skips them)
|
||||||
|
until they are un-suspended. Guards: can't suspend yourself, and can't suspend the last active admin (that would lock admin out). On
|
||||||
un-suspend the user is emailed they can return (suspend is announced reactively, on a blocked
|
un-suspend the user is emailed they can return (suspend is announced reactively, on a blocked
|
||||||
sign-in)."""
|
sign-in)."""
|
||||||
suspended = body.suspended
|
suspended = body.suspended
|
||||||
|
|||||||
+28
-14
@@ -16,6 +16,7 @@ from sqlalchemy import select, update
|
|||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import apitokens
|
||||||
from app import email as email_mod
|
from app import email as email_mod
|
||||||
from app import logs
|
from app import logs
|
||||||
from app import mailtext
|
from app import mailtext
|
||||||
@@ -446,7 +447,7 @@ async def callback(
|
|||||||
# SEC-10: drop any password nobody proved the mailbox for, now that Google attests
|
# SEC-10: drop any password nobody proved the mailbox for, now that Google attests
|
||||||
# ownership (bumps the epoch to revoke any session it minted). Runs before the
|
# ownership (bumps the epoch to revoke any session it minted). Runs before the
|
||||||
# activation below, which is what it reads.
|
# activation below, which is what it reads.
|
||||||
revoke_untrusted_preset_password(user)
|
revoke_untrusted_preset_password(db, user)
|
||||||
# Google verified the email and is_allowed granted access, so finish activating a
|
# Google verified the email and is_allowed granted access, so finish activating a
|
||||||
# previously pending password registration (otherwise current_user would bounce it).
|
# previously pending password registration (otherwise current_user would bounce it).
|
||||||
user.email_verified = True
|
user.email_verified = True
|
||||||
@@ -486,7 +487,7 @@ async def callback(
|
|||||||
# real owner, signed in through Google right now, sets a new password in Settings.
|
# real owner, signed in through Google right now, sets a new password in Settings.
|
||||||
if canonical_email(email) in env_admin_canonicals():
|
if canonical_email(email) in env_admin_canonicals():
|
||||||
if user.role != "admin" and user.password_hash is not None:
|
if user.role != "admin" and user.password_hash is not None:
|
||||||
_revoke_password(user)
|
_revoke_password(db, user)
|
||||||
user.role = "admin"
|
user.role = "admin"
|
||||||
elif not user.role:
|
elif not user.role:
|
||||||
user.role = "user"
|
user.role = "user"
|
||||||
@@ -1118,9 +1119,8 @@ def password_reset_confirm(body: TokenPasswordIn, db: Session = Depends(get_db))
|
|||||||
# about. The person still confirms their address the normal way.
|
# about. The person still confirms their address the normal way.
|
||||||
if kind == "reset":
|
if kind == "reset":
|
||||||
user.email_verified = True
|
user.email_verified = True
|
||||||
bump_session_epoch(
|
# SA4: a reset (often a compromise response) kills all existing sessions — and API tokens.
|
||||||
user
|
bump_session_epoch(db, user)
|
||||||
) # SA4: a reset (often a compromise response) kills all existing sessions.
|
|
||||||
# Burn any other outstanding reset tokens for this user.
|
# Burn any other outstanding reset tokens for this user.
|
||||||
for row in db.execute(
|
for row in db.execute(
|
||||||
select(AuthToken).where(
|
select(AuthToken).where(
|
||||||
@@ -1143,13 +1143,27 @@ def _remember_epoch(request: Request, user: User) -> None:
|
|||||||
request.session["epochs"] = epochs
|
request.session["epochs"] = epochs
|
||||||
|
|
||||||
|
|
||||||
def bump_session_epoch(user: User) -> None:
|
def bump_session_epoch(db: Session, user: User) -> None:
|
||||||
"""Invalidate every EXISTING signed cookie for this account by advancing its epoch (SA4). Cookies
|
"""Invalidate every EXISTING signed cookie for this account by advancing its epoch (SA4). Cookies
|
||||||
minted before the bump carry a now-stale epoch and are rejected by `current_user`."""
|
minted before the bump carry a now-stale epoch and are rejected by `current_user`. The account's
|
||||||
user.session_epoch = (user.session_epoch or 0) + 1
|
API tokens go with them (`apitokens.revoke_all`): a token minted from a stolen session must not
|
||||||
|
outlive the answer to that theft. Not committed — the caller commits it with its change.
|
||||||
|
|
||||||
|
The user's row is locked before the tokens go, the lock `create_token` takes too: a token made at
|
||||||
|
the same moment either commits first and is deleted here, or waits and sees the new epoch. FOR NO
|
||||||
|
KEY UPDATE (`key_share`): the two still exclude each other, but a row that only refers to the user
|
||||||
|
(a notice, an audit line) is not held up — a token's expiry notice would deadlock with it."""
|
||||||
|
if user.id is None:
|
||||||
|
user.session_epoch = (user.session_epoch or 0) + 1
|
||||||
|
return
|
||||||
|
epoch = db.execute(
|
||||||
|
select(User.session_epoch).where(User.id == user.id).with_for_update(key_share=True)
|
||||||
|
).scalar_one()
|
||||||
|
user.session_epoch = (epoch or 0) + 1
|
||||||
|
apitokens.revoke_all(db, user.id)
|
||||||
|
|
||||||
|
|
||||||
def revoke_untrusted_preset_password(user: User) -> None:
|
def revoke_untrusted_preset_password(db: Session, user: User) -> None:
|
||||||
"""SEC-10 (account pre-emption): a password set on an account whose email was NEVER verified is
|
"""SEC-10 (account pre-emption): a password set on an account whose email was NEVER verified is
|
||||||
untrusted — whoever registered the address (to pre-empt the real owner) could have chosen it. When
|
untrusted — whoever registered the address (to pre-empt the real owner) could have chosen it. When
|
||||||
the real owner attests ownership through Google, drop that password and bump the epoch to revoke any
|
the real owner attests ownership through Google, drop that password and bump the epoch to revoke any
|
||||||
@@ -1183,15 +1197,15 @@ def revoke_untrusted_preset_password(user: User) -> None:
|
|||||||
_send_reset_if_eligible keyed off exactly that, so a revoked account silently got no reset link and
|
_send_reset_if_eligible keyed off exactly that, so a revoked account silently got no reset link and
|
||||||
was locked out for good (it could not sign in either). The stamp separates the two cases."""
|
was locked out for good (it could not sign in either). The stamp separates the two cases."""
|
||||||
if not user.email_verified or (not user.is_active and user.password_hash is not None):
|
if not user.email_verified or (not user.is_active and user.password_hash is not None):
|
||||||
_revoke_password(user)
|
_revoke_password(db, user)
|
||||||
|
|
||||||
|
|
||||||
def _revoke_password(user: User) -> None:
|
def _revoke_password(db: Session, user: User) -> None:
|
||||||
"""Drop the account's password, stamp why (so password-reset still accepts it) and end every
|
"""Drop the account's password, stamp why (so password-reset still accepts it) and end every
|
||||||
session it signed in."""
|
session it signed in."""
|
||||||
user.password_hash = None
|
user.password_hash = None
|
||||||
user.password_revoked_at = datetime.now(timezone.utc)
|
user.password_revoked_at = datetime.now(timezone.utc)
|
||||||
bump_session_epoch(user)
|
bump_session_epoch(db, user)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/logout-others")
|
@router.post("/logout-others")
|
||||||
@@ -1203,7 +1217,7 @@ def logout_others(
|
|||||||
in somewhere" / a suspected session compromise."""
|
in somewhere" / a suspected session compromise."""
|
||||||
# Commit the bump BEFORE re-stamping this cookie, so a failed commit can't strand the current
|
# Commit the bump BEFORE re-stamping this cookie, so a failed commit can't strand the current
|
||||||
# session at a newer epoch than the DB (see set_password).
|
# session at a newer epoch than the DB (see set_password).
|
||||||
bump_session_epoch(user)
|
bump_session_epoch(db, user)
|
||||||
db.commit()
|
db.commit()
|
||||||
_remember_epoch(request, user)
|
_remember_epoch(request, user)
|
||||||
log.info("Logout-others: uid=%s (epoch=%s)", user.id, user.session_epoch)
|
log.info("Logout-others: uid=%s (epoch=%s)", user.id, user.session_epoch)
|
||||||
@@ -1255,7 +1269,7 @@ def set_password(
|
|||||||
# SA4: a password change logs out every OTHER session; re-stamp this cookie at the new epoch so
|
# SA4: a password change logs out every OTHER session; re-stamp this cookie at the new epoch so
|
||||||
# the person making the change stays signed in here. Commit BEFORE re-stamping so a failed commit
|
# the person making the change stays signed in here. Commit BEFORE re-stamping so a failed commit
|
||||||
# can't leave the cookie at a newer epoch than the DB (which would wrongly 401 THIS session).
|
# can't leave the cookie at a newer epoch than the DB (which would wrongly 401 THIS session).
|
||||||
bump_session_epoch(user)
|
bump_session_epoch(db, user)
|
||||||
db.commit()
|
db.commit()
|
||||||
_remember_epoch(request, user)
|
_remember_epoch(request, user)
|
||||||
log.info("Password set/changed: uid=%s", user.id)
|
log.info("Password set/changed: uid=%s", user.id)
|
||||||
|
|||||||
+330
-495
File diff suppressed because it is too large
Load Diff
+11
-34
@@ -1,7 +1,7 @@
|
|||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime, timezone
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Query
|
from fastapi import APIRouter, Depends, Query
|
||||||
from pydantic import BaseModel, ConfigDict, Field
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import and_, func, or_, select
|
from sqlalchemy import and_, func, or_, select
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
@@ -26,14 +26,19 @@ from app.feedquery import (
|
|||||||
windowed_relevance_rows,
|
windowed_relevance_rows,
|
||||||
)
|
)
|
||||||
from app.models import (
|
from app.models import (
|
||||||
POSITION_MAX_S,
|
|
||||||
ChannelTag,
|
ChannelTag,
|
||||||
Tag,
|
Tag,
|
||||||
User,
|
User,
|
||||||
Video,
|
Video,
|
||||||
VideoState,
|
VideoState,
|
||||||
)
|
)
|
||||||
from app.routes._common import get_or_404, youtube_refusal, not_found
|
from app.routes._common import (
|
||||||
|
ProgressIn,
|
||||||
|
get_or_404,
|
||||||
|
kept_position,
|
||||||
|
not_found,
|
||||||
|
youtube_refusal,
|
||||||
|
)
|
||||||
from app.sync.videos import parse_iso8601_duration
|
from app.sync.videos import parse_iso8601_duration
|
||||||
from app.utils import iso
|
from app.utils import iso
|
||||||
from app.youtube.client import YouTubeClient, YouTubeError
|
from app.youtube.client import YouTubeClient, YouTubeError
|
||||||
@@ -44,14 +49,6 @@ router = APIRouter(prefix="/api", tags=["feed"])
|
|||||||
# "saved" used to be a status; it's now membership in the built-in Watch later playlist.
|
# "saved" used to be a status; it's now membership in the built-in Watch later playlist.
|
||||||
VALID_STATES = {"new", "watched", "hidden"}
|
VALID_STATES = {"new", "watched", "hidden"}
|
||||||
|
|
||||||
# Resume-position thresholds (mirror the client): positions below this are "didn't
|
|
||||||
# really start" and within this of the end are "basically finished" — neither is worth
|
|
||||||
# storing, so we clear the position instead (the video shows no progress bar).
|
|
||||||
# NB: the Plex player intentionally uses a LARGER finish margin (30s) because it marks
|
|
||||||
# long-form items watched, not just clears a resume bar — see routes/plex/playback.py.
|
|
||||||
PROGRESS_MIN_SECONDS = 5
|
|
||||||
FINISH_MARGIN_SECONDS = 10
|
|
||||||
|
|
||||||
|
|
||||||
# Shared query parameters for /feed and /feed/count.
|
# Shared query parameters for /feed and /feed/count.
|
||||||
def _feed_params(
|
def _feed_params(
|
||||||
@@ -326,23 +323,10 @@ def clear_video_state(
|
|||||||
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
||||||
|
|
||||||
|
|
||||||
class VideoProgressIn(BaseModel):
|
|
||||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
|
||||||
# one into a 500.
|
|
||||||
model_config = ConfigDict(allow_inf_nan=False)
|
|
||||||
|
|
||||||
# Accept a float and truncate (below) — the player may post a raw currentTime (a float), which
|
|
||||||
# the old `int(payload.get(...))` silently truncated. A plain `int` field would 422 that on the
|
|
||||||
# 10s progress hot path, losing the resume position; `float` keeps the old tolerance.
|
|
||||||
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
|
||||||
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
|
||||||
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/videos/{video_id}/progress")
|
@router.post("/videos/{video_id}/progress")
|
||||||
def set_video_progress(
|
def set_video_progress(
|
||||||
video_id: str,
|
video_id: str,
|
||||||
body: VideoProgressIn,
|
body: ProgressIn,
|
||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
@@ -350,22 +334,15 @@ def set_video_progress(
|
|||||||
Stores a per-user position on the video_states row without touching watch status, so a
|
Stores a per-user position on the video_states row without touching watch status, so a
|
||||||
partially-watched video can render a progress bar and match the "in progress" filter.
|
partially-watched video can render a progress bar and match the "in progress" filter.
|
||||||
Trivially-early and near-finished positions clear the position rather than store it."""
|
Trivially-early and near-finished positions clear the position rather than store it."""
|
||||||
position = int(body.position_seconds)
|
position = kept_position(body)
|
||||||
duration = int(body.duration_seconds)
|
|
||||||
if position < 0:
|
|
||||||
position = 0
|
|
||||||
|
|
||||||
get_or_404(db, Video, video_id, what="video")
|
get_or_404(db, Video, video_id, what="video")
|
||||||
|
|
||||||
# Decide whether this position is worth keeping (else clear it).
|
|
||||||
near_end = duration > 0 and position > duration - FINISH_MARGIN_SECONDS
|
|
||||||
keep = position >= PROGRESS_MIN_SECONDS and not near_end
|
|
||||||
|
|
||||||
row = db.execute(
|
row = db.execute(
|
||||||
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
|
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
|
|
||||||
if not keep:
|
if not position:
|
||||||
# Nothing meaningful to store: drop a status-less row entirely, otherwise just
|
# Nothing meaningful to store: drop a status-less row entirely, otherwise just
|
||||||
# zero the position (a saved/hidden video keeps its status).
|
# zero the position (a saved/hidden video keeps its status).
|
||||||
if row is not None:
|
if row is not None:
|
||||||
|
|||||||
@@ -0,0 +1,945 @@
|
|||||||
|
"""The Files library HTTP API — a user's files, the folders they sit in, and the file itself.
|
||||||
|
|
||||||
|
A file in the library is a user's DONE job (`DownloadJob`) over its shared `MediaAsset`: the job is
|
||||||
|
the per-user layer (name, folder, kept, expiry, shares and links all hang off it), the asset the
|
||||||
|
bytes. Folders are rows (`downloads.folders`), so a rename or a move never touches the disk.
|
||||||
|
|
||||||
|
In-flight jobs are not here: they are the download tray's (`routes.downloads`). The helpers above
|
||||||
|
the routes are shared with that module — serialization, the ownership locks, the upload spool —
|
||||||
|
because a download and a library file are one row at two moments.
|
||||||
|
|
||||||
|
A machine with an API token (`app.apitokens`) may use the routes that take `files_caller`: list the
|
||||||
|
folders and files, fetch a file and its hash, upload, delete — each only in a folder its token was
|
||||||
|
granted that right in, and never the files shared with the user. A folder or file out of its reach
|
||||||
|
answers like a missing one. Everything else here is the signed-in user's alone (`current_user`).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
from collections.abc import AsyncIterator, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import BinaryIO, Literal
|
||||||
|
from urllib.parse import unquote, urlparse
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||||
|
from fastapi.responses import FileResponse, Response
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from sqlalchemy import func, or_, select, update
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import audit, sysconfig
|
||||||
|
from app.apitokens import FilesCaller, Right, files_caller
|
||||||
|
from app.audit import AuditAction
|
||||||
|
from app.auth import current_user
|
||||||
|
from app.collation import collated
|
||||||
|
from app.config import settings
|
||||||
|
from app.db import get_db, release_read_connection
|
||||||
|
from app.downloads import folders, preview, quota, service, storage
|
||||||
|
from app.downloads import upload as uploadmod
|
||||||
|
from app.downloads.status import JobStatus
|
||||||
|
from app.mailtext import lang_for_user
|
||||||
|
from app.models import (
|
||||||
|
DISPLAY_TEXT_MAX,
|
||||||
|
DownloadJob,
|
||||||
|
DownloadLink,
|
||||||
|
DownloadShare,
|
||||||
|
FileFolder,
|
||||||
|
LibraryState,
|
||||||
|
MediaAsset,
|
||||||
|
User,
|
||||||
|
display_text,
|
||||||
|
)
|
||||||
|
from app.refusal import refused
|
||||||
|
from app.routes._common import capped_stream_to, not_found, owned_or_404
|
||||||
|
from app.utils import iso, like_escape
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/files", tags=["files"])
|
||||||
|
|
||||||
|
# Cap the listing so a heavy library can't return an unbounded result set (the old Download Center's
|
||||||
|
# own cap; far beyond any real library).
|
||||||
|
MAX_LIST = 500
|
||||||
|
# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
|
||||||
|
_MAX_RETENTION_DAYS = 3650
|
||||||
|
_HASH_CHUNK = 1 << 20
|
||||||
|
# How many files the hash route reads at once: each holds a threadpool thread for the whole read.
|
||||||
|
_HASHERS = 2
|
||||||
|
_HASHING = threading.BoundedSemaphore(_HASHERS)
|
||||||
|
|
||||||
|
|
||||||
|
# --- shared with routes.downloads -------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||||
|
ready = asset is not None and asset.status == "ready" and bool(asset.rel_path)
|
||||||
|
return {
|
||||||
|
"id": job.id,
|
||||||
|
"status": job.status,
|
||||||
|
"progress": job.progress,
|
||||||
|
"speed_bps": job.speed_bps,
|
||||||
|
"eta_s": job.eta_s,
|
||||||
|
"phase": job.phase,
|
||||||
|
"error": job.error,
|
||||||
|
"error_code": job.error_code,
|
||||||
|
"error_detail": job.error_detail,
|
||||||
|
"queue_pos": job.queue_pos,
|
||||||
|
"created_at": iso(job.created_at),
|
||||||
|
"ready_at": iso(job.ready_at),
|
||||||
|
"source_kind": job.source_kind,
|
||||||
|
"source_ref": job.source_ref,
|
||||||
|
"source_url": service.reference_url(job, asset),
|
||||||
|
# A locally-generated poster to fall back to when the source had no thumbnail.
|
||||||
|
"poster_url": f"/api/downloads/{job.id}/poster.jpg"
|
||||||
|
if (asset is not None and asset.poster_path)
|
||||||
|
else None,
|
||||||
|
"profile_id": job.profile_id,
|
||||||
|
"spec": job.profile_snapshot,
|
||||||
|
"display_name": job.display_name,
|
||||||
|
"display_uploader": job.display_uploader,
|
||||||
|
"display_uploader_url": job.display_uploader_url,
|
||||||
|
"extra_links": job.extra_links or [],
|
||||||
|
"job_kind": job.job_kind,
|
||||||
|
"source_job_id": job.source_job_id,
|
||||||
|
"edit_spec": job.edit_spec,
|
||||||
|
"can_download": ready and job.status == JobStatus.DONE,
|
||||||
|
# The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
|
||||||
|
"folder_id": job.folder_id,
|
||||||
|
"kept": job.kept,
|
||||||
|
"expires_at": iso(job.expires_at),
|
||||||
|
"asset": None
|
||||||
|
if asset is None
|
||||||
|
else {
|
||||||
|
"id": asset.id,
|
||||||
|
"status": asset.status,
|
||||||
|
"title": asset.title,
|
||||||
|
"uploader": asset.uploader,
|
||||||
|
"upload_date": iso(asset.upload_date),
|
||||||
|
"duration_s": asset.duration_s,
|
||||||
|
"size_bytes": asset.size_bytes,
|
||||||
|
"width": asset.width,
|
||||||
|
"height": asset.height,
|
||||||
|
"container": asset.container,
|
||||||
|
# "video" | "audio" | "file" — the library gates play/edit on this (media only); a
|
||||||
|
# generic uploaded file is download-only. Null while still in flight (set at finalize).
|
||||||
|
"media_kind": asset.media_kind,
|
||||||
|
"uploader_url": asset.uploader_url,
|
||||||
|
"thumbnail_url": asset.thumbnail_url,
|
||||||
|
# "image" | "pdf" | "text" | null — what `/preview` shows (`downloads/preview.py`).
|
||||||
|
"preview": preview.kind(asset.media_kind, asset.container),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def assets_for(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, MediaAsset]:
|
||||||
|
ids = {j.asset_id for j in jobs if j.asset_id}
|
||||||
|
if not ids:
|
||||||
|
return {}
|
||||||
|
rows = db.execute(select(MediaAsset).where(MediaAsset.id.in_(ids))).scalars()
|
||||||
|
return {a.id: a for a in rows}
|
||||||
|
|
||||||
|
|
||||||
|
def positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
|
||||||
|
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
|
||||||
|
ids = [j.id for j in jobs]
|
||||||
|
if not ids:
|
||||||
|
return {}
|
||||||
|
rows = db.execute(
|
||||||
|
select(LibraryState.job_id, LibraryState.position_seconds).where(
|
||||||
|
LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return {job_id: pos for job_id, pos in rows}
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_job(db: Session, job: DownloadJob) -> dict:
|
||||||
|
"""Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
|
||||||
|
every single-job handler shares."""
|
||||||
|
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||||
|
return serialize(job, asset)
|
||||||
|
|
||||||
|
|
||||||
|
def own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||||
|
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
||||||
|
or writes the status those decide on (pause). The job is re-read under a row lock, so two
|
||||||
|
overlapping requests for one job run one after the other and the second decides on the first's
|
||||||
|
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||||
|
just released back into a holder; either can free a file another job still holds.
|
||||||
|
|
||||||
|
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||||
|
(`service.finalize_if_running`), so the two can never deadlock."""
|
||||||
|
while True:
|
||||||
|
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
|
||||||
|
if asset_id is not None:
|
||||||
|
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||||
|
job = db.execute(
|
||||||
|
select(DownloadJob)
|
||||||
|
.where(DownloadJob.id == job_id)
|
||||||
|
.with_for_update()
|
||||||
|
.execution_options(populate_existing=True)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if job is None:
|
||||||
|
raise not_found("download")
|
||||||
|
if job.asset_id == asset_id:
|
||||||
|
return job
|
||||||
|
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||||
|
|
||||||
|
|
||||||
|
def quota_detail(e: quota.QuotaExceeded) -> dict:
|
||||||
|
"""Structured error the trilingual client localizes (reason key + numbers), instead of a
|
||||||
|
pre-baked English string with a dot-decimal GB value. See lib/api.ts localizeDetail()."""
|
||||||
|
return {"code": "quota", "reason": e.reason, "limit": e.limit, "current": e.current}
|
||||||
|
|
||||||
|
|
||||||
|
# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
|
||||||
|
# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
|
||||||
|
# FastAPI *before the first dependency runs*, so the auth dependency could not gate it and an anonymous
|
||||||
|
# caller could make the process write the whole per-file allowance to temp, per request (SEC-20
|
||||||
|
# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
|
||||||
|
# declared AFTER the auth one, is the only shape that closes it — the same one
|
||||||
|
# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
|
||||||
|
# every route with a raised ceiling rather than leaving it to a comment like this one.
|
||||||
|
#
|
||||||
|
# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a
|
||||||
|
# Hungarian filename would otherwise be unsendable.
|
||||||
|
UPLOAD_FILENAME_HEADER = "x-upload-filename"
|
||||||
|
UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name"
|
||||||
|
# The folder the file goes in (its id); without it, the user's Uploads folder.
|
||||||
|
UPLOAD_FOLDER_HEADER = "x-upload-folder"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SpooledUpload:
|
||||||
|
"""What the body dependency hands the handler: the bytes already on disk and their hash, plus
|
||||||
|
the names and the folder that came alongside them, and who sent them."""
|
||||||
|
|
||||||
|
path: Path
|
||||||
|
size: int
|
||||||
|
sha256: str
|
||||||
|
user_id: int
|
||||||
|
filename: str
|
||||||
|
display_name: str | None
|
||||||
|
folder_id: int | None
|
||||||
|
via: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _header_text(request: Request, name: str) -> str:
|
||||||
|
"""One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the
|
||||||
|
caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's
|
||||||
|
own fallback names the file."""
|
||||||
|
raw = request.headers.get(name, "")
|
||||||
|
try:
|
||||||
|
return unquote(raw, errors="strict").strip()
|
||||||
|
except (UnicodeDecodeError, ValueError):
|
||||||
|
# silent-ok: a mis-encoded header is the caller's own malformed request, and this one is
|
||||||
|
# cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on
|
||||||
|
# a label; logging it would hand any client a free log line per request.
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _upload_folder(db: Session, caller: FilesCaller, raw: str) -> int | None:
|
||||||
|
"""The folder id an upload asked for (`UPLOAD_FOLDER_HEADER`), checked; None when it asked for
|
||||||
|
none — which a token may not: it writes only where it was granted to."""
|
||||||
|
if not raw.strip():
|
||||||
|
if caller.grants is not None:
|
||||||
|
raise refused(400, "api_token_folder_required")
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
folder_id = int(raw)
|
||||||
|
except ValueError as e:
|
||||||
|
raise refused(409, "file_folder_gone") from e
|
||||||
|
if not caller.may(folder_id, Right.WRITE):
|
||||||
|
raise not_found("folder")
|
||||||
|
try:
|
||||||
|
return folders.upload_target(db, caller.user.id, folder_id, hold=False).id
|
||||||
|
except folders.FolderError as e:
|
||||||
|
raise refused(409, e.reason) from e
|
||||||
|
|
||||||
|
|
||||||
|
class _HashingSink:
|
||||||
|
"""The upload's file, hashing the bytes on their way in: the hash costs no second read."""
|
||||||
|
|
||||||
|
def __init__(self, out: BinaryIO) -> None:
|
||||||
|
self.out = out
|
||||||
|
self.hash = hashlib.sha256()
|
||||||
|
|
||||||
|
def write(self, data: bytes, /) -> int:
|
||||||
|
self.hash.update(data)
|
||||||
|
return self.out.write(data)
|
||||||
|
|
||||||
|
|
||||||
|
async def spool_upload(
|
||||||
|
request: Request,
|
||||||
|
caller: FilesCaller = Depends(files_caller),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> AsyncIterator[SpooledUpload]:
|
||||||
|
"""Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
|
||||||
|
|
||||||
|
`caller` comes first on purpose. Dependencies resolve in declaration order, so naming
|
||||||
|
`files_caller` here makes the auth check a precondition of reading the body at all — a stranger,
|
||||||
|
or a token without the right to write in that folder, is refused before a byte moves. The
|
||||||
|
`yield` shape owns the temp file's life: ingest MOVES it into place on success (the unlink is
|
||||||
|
then a no-op), and on any other exit — cap hit, empty, quota, probe or move error, a client that
|
||||||
|
vanished mid-upload — it is removed."""
|
||||||
|
cap = sysconfig.get_int(db, "download_upload_max_bytes")
|
||||||
|
user_id = caller.user.id
|
||||||
|
via = caller.via
|
||||||
|
# A folder the file may not go in is refused before the transfer, not after it.
|
||||||
|
folder_id = _upload_folder(db, caller, request.headers.get(UPLOAD_FOLDER_HEADER, ""))
|
||||||
|
# Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection`
|
||||||
|
# for why, and for the precondition it enforces. Everything this dependency still needs is
|
||||||
|
# already an int, a string or a header in hand: touching an ORM attribute below (`caller.user`)
|
||||||
|
# would reload it and check a connection straight back out.
|
||||||
|
release_read_connection(db)
|
||||||
|
staging = Path(settings.files_root) / ".staging" / "uploads"
|
||||||
|
staging.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, tmp = tempfile.mkstemp(dir=str(staging))
|
||||||
|
tmp_path = Path(tmp)
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "wb") as out:
|
||||||
|
sink = _HashingSink(out)
|
||||||
|
size = await capped_stream_to(
|
||||||
|
request,
|
||||||
|
sink,
|
||||||
|
cap,
|
||||||
|
HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}),
|
||||||
|
)
|
||||||
|
yield SpooledUpload(
|
||||||
|
path=tmp_path,
|
||||||
|
size=size,
|
||||||
|
sha256=sink.hash.hexdigest(),
|
||||||
|
user_id=user_id,
|
||||||
|
filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload",
|
||||||
|
display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)),
|
||||||
|
folder_id=folder_id,
|
||||||
|
via=via,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
tmp_path.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||||
|
"""A file the caller may read: their own, or one shared with them."""
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
|
if job is None:
|
||||||
|
raise not_found("download")
|
||||||
|
if job.user_id == user.id:
|
||||||
|
return job
|
||||||
|
shared = db.execute(
|
||||||
|
select(DownloadShare.id).where(
|
||||||
|
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
|
||||||
|
)
|
||||||
|
).first()
|
||||||
|
if shared is None:
|
||||||
|
raise not_found("download")
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def readable_job(db: Session, caller: FilesCaller, job_id: int) -> DownloadJob:
|
||||||
|
"""A file `caller` may read: as `accessible_job` for the user; for a token, one of the user's
|
||||||
|
own in a folder it may read."""
|
||||||
|
if caller.grants is None:
|
||||||
|
return accessible_job(db, caller.user, job_id)
|
||||||
|
job = db.get(DownloadJob, job_id)
|
||||||
|
if job is None or job.user_id != caller.user.id or not caller.may(job.folder_id, Right.READ):
|
||||||
|
raise not_found("download")
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
# --- folders -------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_folder(folder: FileFolder, count: int, size: int, linked: bool = False) -> dict:
|
||||||
|
return {
|
||||||
|
"id": folder.id,
|
||||||
|
"kind": folder.kind,
|
||||||
|
# A source folder's name is the client's to render from its key, in the reader's language.
|
||||||
|
"source_key": folder.source_key,
|
||||||
|
"name": folder.name,
|
||||||
|
"retention_days": folder.retention_days,
|
||||||
|
"count": count,
|
||||||
|
"size_bytes": size,
|
||||||
|
"created_at": iso(folder.created_at),
|
||||||
|
# A live public link opens it: whatever is moved in is shown to whoever holds the link.
|
||||||
|
"has_link": linked,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _linked(db: Session, folder_ids: list[int]) -> set[int]:
|
||||||
|
"""Which of these folders a live public link opens (one not past its expiry)."""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
rows = db.execute(
|
||||||
|
select(DownloadLink.folder_id).where(
|
||||||
|
DownloadLink.folder_id.in_(folder_ids),
|
||||||
|
or_(DownloadLink.expires_at.is_(None), DownloadLink.expires_at > now),
|
||||||
|
)
|
||||||
|
).scalars()
|
||||||
|
return {folder_id for folder_id in rows if folder_id is not None}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/folders")
|
||||||
|
def list_folders(
|
||||||
|
caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
||||||
|
) -> list[dict]:
|
||||||
|
"""The caller's folders with how many files each holds and their size (a token: the ones it may
|
||||||
|
list). A source folder its files have all left stays (count 0): the client decides whether to
|
||||||
|
show it."""
|
||||||
|
done = (DownloadJob.folder_id == FileFolder.id) & (DownloadJob.status == JobStatus.DONE)
|
||||||
|
stmt = select(
|
||||||
|
FileFolder,
|
||||||
|
func.count(DownloadJob.id),
|
||||||
|
func.coalesce(func.sum(MediaAsset.size_bytes), 0),
|
||||||
|
).where(FileFolder.user_id == caller.user.id)
|
||||||
|
listable = caller.folders_with(Right.LIST)
|
||||||
|
if listable is not None:
|
||||||
|
stmt = stmt.where(FileFolder.id.in_(listable))
|
||||||
|
rows = db.execute(
|
||||||
|
stmt.outerjoin(DownloadJob, done)
|
||||||
|
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||||
|
.group_by(FileFolder.id)
|
||||||
|
.order_by(FileFolder.kind.desc(), FileFolder.source_key, FileFolder.name, FileFolder.id)
|
||||||
|
).all()
|
||||||
|
linked = _linked(db, [f.id for f, _, _ in rows])
|
||||||
|
return [_serialize_folder(f, count, int(size), f.id in linked) for f, count, size in rows]
|
||||||
|
|
||||||
|
|
||||||
|
class FolderIn(BaseModel):
|
||||||
|
name: str = Field(min_length=1, max_length=DISPLAY_TEXT_MAX)
|
||||||
|
|
||||||
|
|
||||||
|
def _folder_name(raw: str) -> str:
|
||||||
|
name = display_text(raw)
|
||||||
|
if not name:
|
||||||
|
raise refused(400, "file_folder_name_required")
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/folders")
|
||||||
|
def create_folder(
|
||||||
|
body: FolderIn, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||||
|
) -> dict:
|
||||||
|
folder = FileFolder(user_id=user.id, kind=folders.FolderKind.OWN, name=_folder_name(body.name))
|
||||||
|
try:
|
||||||
|
with db.begin_nested():
|
||||||
|
db.add(folder)
|
||||||
|
db.flush()
|
||||||
|
except IntegrityError as e: # the unique (user, lower(name)): one folder per name
|
||||||
|
raise refused(409, "file_folder_name_taken") from e
|
||||||
|
audit.record(
|
||||||
|
db, user.id, AuditAction.FILE_FOLDER_CREATE, target_type="file_folder", target_id=folder.id
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return _serialize_folder(folder, 0, 0)
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateFolderIn(BaseModel):
|
||||||
|
# Presence (model_fields_set) decides what is touched; retention_days null = keep forever.
|
||||||
|
name: str | None = Field(default=None, min_length=1, max_length=DISPLAY_TEXT_MAX)
|
||||||
|
retention_days: int | None = Field(default=None, ge=1, le=_MAX_RETENTION_DAYS)
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/folders/{folder_id}")
|
||||||
|
def update_folder(
|
||||||
|
folder_id: int,
|
||||||
|
body: UpdateFolderIn,
|
||||||
|
user: User = Depends(current_user),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
"""Rename one of the caller's own folders (a source folder's name is its origin's), and/or set
|
||||||
|
how long any of their folders keeps a file (`folders.set_retention`)."""
|
||||||
|
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||||
|
sent = body.model_fields_set
|
||||||
|
if "name" in sent:
|
||||||
|
if folder.kind != folders.FolderKind.OWN:
|
||||||
|
raise refused(409, "file_folder_not_own")
|
||||||
|
folder.name = _folder_name(body.name or "")
|
||||||
|
try:
|
||||||
|
with db.begin_nested():
|
||||||
|
db.flush()
|
||||||
|
except IntegrityError as e:
|
||||||
|
raise refused(409, "file_folder_name_taken") from e
|
||||||
|
audit.record(
|
||||||
|
db,
|
||||||
|
user.id,
|
||||||
|
AuditAction.FILE_FOLDER_RENAME,
|
||||||
|
target_type="file_folder",
|
||||||
|
target_id=folder.id,
|
||||||
|
)
|
||||||
|
if "retention_days" in sent:
|
||||||
|
# The value replaced is the one read under the folder's lock, not the one read above it.
|
||||||
|
try:
|
||||||
|
before = folders.set_retention(db, folder, body.retention_days)
|
||||||
|
except folders.FolderError as e:
|
||||||
|
raise refused(409, e.reason) from e
|
||||||
|
if before != body.retention_days:
|
||||||
|
audit.record(
|
||||||
|
db,
|
||||||
|
user.id,
|
||||||
|
AuditAction.FILE_FOLDER_RETENTION,
|
||||||
|
target_type="file_folder",
|
||||||
|
target_id=folder.id,
|
||||||
|
before={"retention_days": before},
|
||||||
|
after={"retention_days": body.retention_days},
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
count, size = _folder_totals(db, folder.id)
|
||||||
|
return _serialize_folder(folder, count, size, folder.id in _linked(db, [folder.id]))
|
||||||
|
|
||||||
|
|
||||||
|
def _folder_totals(db: Session, folder_id: int) -> tuple[int, int]:
|
||||||
|
count, size = db.execute(
|
||||||
|
select(func.count(DownloadJob.id), func.coalesce(func.sum(MediaAsset.size_bytes), 0))
|
||||||
|
.select_from(DownloadJob)
|
||||||
|
.outerjoin(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||||
|
.where(DownloadJob.folder_id == folder_id, DownloadJob.status == JobStatus.DONE)
|
||||||
|
).one()
|
||||||
|
return count, int(size)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/folders/{folder_id}")
|
||||||
|
def delete_folder(
|
||||||
|
folder_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||||
|
) -> dict:
|
||||||
|
"""Delete one of the caller's own folders; its files go back to their source folders."""
|
||||||
|
folder = owned_or_404(db, FileFolder, folder_id, user, what="folder")
|
||||||
|
try:
|
||||||
|
moved = folders.delete_own(db, folder)
|
||||||
|
except folders.FolderError as e:
|
||||||
|
raise refused(409, e.reason) from e
|
||||||
|
audit.record(
|
||||||
|
db,
|
||||||
|
user.id,
|
||||||
|
AuditAction.FILE_FOLDER_DELETE,
|
||||||
|
target_type="file_folder",
|
||||||
|
target_id=folder_id,
|
||||||
|
after={"files_moved": moved},
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return {"ok": True, "deleted": folder_id, "files_moved": moved}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/usage")
|
||||||
|
def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||||
|
"""The caller's storage quota, plus the per-file upload cap.
|
||||||
|
|
||||||
|
The cap rides here rather than in `quota.usage` because it is not a quota value — it is a
|
||||||
|
`sysconfig` knob about one route — and this is the route layer, where composing the two is the
|
||||||
|
job. The upload dialog needs it to refuse an over-cap file BEFORE sending it: both server-side
|
||||||
|
caps answer too late to be pleasant (the edge ceiling refuses the request outright, see
|
||||||
|
`bodylimit.ROUTE_MAX_BODY_BYTES`), and making the user push 600 MB up the wire to be told the
|
||||||
|
limit is 500 is a bad answer even when it is a correct one. It is a convenience, not the
|
||||||
|
boundary: the caps stay where they are and still refuse a client that ignores this.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
**quota.usage(db, user.id),
|
||||||
|
"upload_max_bytes": sysconfig.get_int(db, "download_upload_max_bytes"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/shared/{job_id}")
|
||||||
|
def remove_shared_with_me(
|
||||||
|
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||||
|
) -> dict:
|
||||||
|
"""Remove a file that was shared WITH this user from their 'Shared with me' list. Deletes
|
||||||
|
only the recipient's share grant — the owner's job and the physical file are untouched (this is
|
||||||
|
a per-user dismissal, not a delete)."""
|
||||||
|
row = db.execute(
|
||||||
|
select(DownloadShare).where(
|
||||||
|
DownloadShare.job_id == job_id, DownloadShare.to_user_id == user.id
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if row is not None:
|
||||||
|
db.delete(row)
|
||||||
|
db.commit()
|
||||||
|
return {"ok": True, "removed": job_id}
|
||||||
|
|
||||||
|
|
||||||
|
# --- files ---------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("")
|
||||||
|
def list_files(
|
||||||
|
folder_id: int | None = None,
|
||||||
|
q: str = Query("", max_length=200),
|
||||||
|
sort: Literal["added", "name", "size"] = "added",
|
||||||
|
order: Literal["asc", "desc"] = "desc",
|
||||||
|
scope: Literal["mine", "shared"] = "mine",
|
||||||
|
caller: FilesCaller = Depends(files_caller),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> list[dict]:
|
||||||
|
"""The caller's done files (`scope=mine`, optionally one folder's), or the ones shared with
|
||||||
|
them (`scope=shared`); searched by name and channel, sorted by when they were added, by name
|
||||||
|
(in the reader's alphabet) or by size. A token lists only in the folders it may list, and never
|
||||||
|
the shared ones."""
|
||||||
|
user = caller.user
|
||||||
|
listable = caller.folders_with(Right.LIST)
|
||||||
|
if listable is not None:
|
||||||
|
if scope == "shared":
|
||||||
|
raise refused(403, "api_token_scope")
|
||||||
|
if folder_id is not None and folder_id not in listable:
|
||||||
|
raise not_found("folder")
|
||||||
|
stmt = (
|
||||||
|
select(DownloadJob)
|
||||||
|
.join(MediaAsset, MediaAsset.id == DownloadJob.asset_id)
|
||||||
|
.where(DownloadJob.status == JobStatus.DONE)
|
||||||
|
)
|
||||||
|
shared = scope == "shared"
|
||||||
|
if shared:
|
||||||
|
stmt = stmt.join(DownloadShare, DownloadShare.job_id == DownloadJob.id).where(
|
||||||
|
DownloadShare.to_user_id == user.id
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
stmt = stmt.where(DownloadJob.user_id == user.id)
|
||||||
|
if folder_id is not None:
|
||||||
|
stmt = stmt.where(DownloadJob.folder_id == folder_id)
|
||||||
|
elif listable is not None:
|
||||||
|
stmt = stmt.where(DownloadJob.folder_id.in_(listable))
|
||||||
|
name = func.coalesce(DownloadJob.display_name, MediaAsset.title, "")
|
||||||
|
term = q.strip()
|
||||||
|
if term:
|
||||||
|
like = f"%{like_escape(term)}%"
|
||||||
|
stmt = stmt.where(
|
||||||
|
or_(
|
||||||
|
name.ilike(like),
|
||||||
|
func.coalesce(DownloadJob.display_uploader, MediaAsset.uploader, "").ilike(like),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# A file is added to its owner's list when it is done, not when it was asked for (a download that
|
||||||
|
# failed for days and then went through is new); one shared with the caller, when it was shared.
|
||||||
|
key = {
|
||||||
|
"added": DownloadShare.created_at if shared else DownloadJob.ready_at,
|
||||||
|
"name": collated(name, lang_for_user(user)),
|
||||||
|
"size": MediaAsset.size_bytes,
|
||||||
|
}[sort]
|
||||||
|
first = key.asc() if order == "asc" else key.desc()
|
||||||
|
tie = DownloadJob.id.asc() if order == "asc" else DownloadJob.id.desc()
|
||||||
|
jobs = db.execute(stmt.order_by(first.nulls_last(), tie).limit(MAX_LIST)).scalars().all()
|
||||||
|
assets = assets_for(db, jobs)
|
||||||
|
positions = positions_for(db, user, jobs)
|
||||||
|
shared_at = _shared_at(db, user, jobs) if shared else {}
|
||||||
|
sources = source_keys(db, jobs)
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
**serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
|
||||||
|
"position_seconds": positions.get(j.id, 0),
|
||||||
|
"shared": shared,
|
||||||
|
"shared_at": iso(shared_at.get(j.id)),
|
||||||
|
"source_key": sources[j.id],
|
||||||
|
}
|
||||||
|
for j in jobs
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def source_keys(db: Session, jobs: Sequence[DownloadJob]) -> dict[int, str]:
|
||||||
|
"""Where each file came from (its source folder's key), for the list's Source column, the
|
||||||
|
"back to its source folder" move and the download tray's site: a file in a source folder has its folder's key; one moved into
|
||||||
|
an own folder (or someone else's, shared) is traced back by `folders.source_key_for`."""
|
||||||
|
ids = {j.folder_id for j in jobs if j.folder_id is not None}
|
||||||
|
by_id = (
|
||||||
|
{f.id: f for f in db.execute(select(FileFolder).where(FileFolder.id.in_(ids))).scalars()}
|
||||||
|
if ids
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
out: dict[int, str] = {}
|
||||||
|
for j in jobs:
|
||||||
|
folder = by_id.get(j.folder_id) if j.folder_id is not None else None
|
||||||
|
if folder is not None and folder.kind == folders.FolderKind.SOURCE and folder.source_key:
|
||||||
|
out[j.id] = folder.source_key
|
||||||
|
else:
|
||||||
|
out[j.id] = folders.source_key_for(db, j)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _shared_at(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, datetime]:
|
||||||
|
rows = db.execute(
|
||||||
|
select(DownloadShare.job_id, DownloadShare.created_at).where(
|
||||||
|
DownloadShare.to_user_id == user.id,
|
||||||
|
DownloadShare.job_id.in_([j.id for j in jobs]),
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return {job_id: at for job_id, at in rows}
|
||||||
|
|
||||||
|
|
||||||
|
_MAX_EXTRA_LINKS = 8
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_url(raw) -> str | None:
|
||||||
|
"""Accept only a plausible http(s) URL (these get rendered as clickable links + embedded in a
|
||||||
|
public watch page); anything else is dropped rather than trusted."""
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
return None
|
||||||
|
url = raw.strip()
|
||||||
|
if not url:
|
||||||
|
return None
|
||||||
|
if not url.startswith(("http://", "https://")):
|
||||||
|
return None
|
||||||
|
# `urlparse` raises on a malformed authority; this function already has a "not a plausible URL"
|
||||||
|
# answer, so the typo belongs in it.
|
||||||
|
try:
|
||||||
|
parsed = urlparse(url)
|
||||||
|
# silent-ok: a typo in a link someone pasted. None is this function's "not a plausible URL"
|
||||||
|
# answer, the caller already has one, and any line here would carry the URL itself.
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
if not parsed.netloc:
|
||||||
|
return None
|
||||||
|
return url[:2048]
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateFileIn(BaseModel):
|
||||||
|
# Presence (model_fields_set) decides what is touched; a null/"" text falls back to the system
|
||||||
|
# value (the title, the channel). All display-only: the file on disk keeps its own name.
|
||||||
|
name: str | None = None
|
||||||
|
uploader: str | None = None
|
||||||
|
uploader_url: str | None = None
|
||||||
|
extra_links: list | None = None
|
||||||
|
folder_id: int | None = None
|
||||||
|
kept: bool | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/{job_id}")
|
||||||
|
def update_file(
|
||||||
|
job_id: int,
|
||||||
|
body: UpdateFileIn,
|
||||||
|
user: User = Depends(current_user),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
"""Rename a file and its channel, set its reference links (all display-only: the file on disk
|
||||||
|
keeps its own name), move it to another folder (`folders.move`), or keep it until deleted /
|
||||||
|
hand it back to its folder's retention."""
|
||||||
|
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||||
|
sent = body.model_fields_set
|
||||||
|
if "name" in sent:
|
||||||
|
job.display_name = display_text(body.name)
|
||||||
|
if "uploader" in sent:
|
||||||
|
job.display_uploader = display_text(body.uploader)
|
||||||
|
if "uploader_url" in sent:
|
||||||
|
job.display_uploader_url = _clean_url(body.uploader_url)
|
||||||
|
if "extra_links" in sent:
|
||||||
|
raw = body.extra_links or []
|
||||||
|
cleaned = [u for u in (_clean_url(x) for x in raw) if u][:_MAX_EXTRA_LINKS]
|
||||||
|
job.extra_links = cleaned or None
|
||||||
|
if "folder_id" in sent:
|
||||||
|
if body.folder_id is None:
|
||||||
|
raise refused(400, "file_folder_gone")
|
||||||
|
before = job.folder_id
|
||||||
|
try:
|
||||||
|
folders.move(db, job, folders.own_folder(db, user.id, body.folder_id))
|
||||||
|
except folders.FolderError as e:
|
||||||
|
raise refused(409, e.reason) from e
|
||||||
|
if job.folder_id != before:
|
||||||
|
audit.record(
|
||||||
|
db,
|
||||||
|
user.id,
|
||||||
|
AuditAction.FILE_MOVE,
|
||||||
|
target_type="file",
|
||||||
|
target_id=job.id,
|
||||||
|
before={"folder_id": before},
|
||||||
|
after={"folder_id": job.folder_id},
|
||||||
|
)
|
||||||
|
if "kept" in sent and body.kept is not None and body.kept != job.kept:
|
||||||
|
folders.set_kept(db, job, body.kept)
|
||||||
|
action = AuditAction.FILE_KEEP if body.kept else AuditAction.FILE_UNKEEP
|
||||||
|
audit.record(db, user.id, action, target_type="file", target_id=job.id)
|
||||||
|
db.commit()
|
||||||
|
return serialize_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{job_id}")
|
||||||
|
def delete_file(
|
||||||
|
job_id: int, caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
||||||
|
) -> dict:
|
||||||
|
"""Delete a file from the caller's library; the bytes go with their last holder. A token
|
||||||
|
deletes only in a folder it may delete in — judged on the row as locked, so a file moved out of
|
||||||
|
that folder meanwhile is out of its reach."""
|
||||||
|
job = own_job_for_update(db, caller.user, job_id)
|
||||||
|
if not caller.may(job.folder_id, Right.DELETE):
|
||||||
|
db.rollback() # lets go of the locks; nothing was written
|
||||||
|
raise not_found("download")
|
||||||
|
audit.record(
|
||||||
|
db,
|
||||||
|
caller.user.id,
|
||||||
|
AuditAction.FILE_DELETE,
|
||||||
|
target_type="file",
|
||||||
|
target_id=job_id,
|
||||||
|
summary=caller.via,
|
||||||
|
)
|
||||||
|
service.delete(db, job) # commits, the audit row with it
|
||||||
|
return {"ok": True, "deleted": job_id}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/upload")
|
||||||
|
def upload_file(
|
||||||
|
upload: SpooledUpload = Depends(spool_upload),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
"""Put a local file in the caller's library — in the folder `UPLOAD_FOLDER_HEADER` names, else
|
||||||
|
their Uploads folder. Any file type; media gets the player, a generic file is download-only.
|
||||||
|
The body IS the file (see the header note above `UPLOAD_FILENAME_HEADER`); `spool_upload` has
|
||||||
|
authenticated the caller, checked the folder and streamed it to disk already.
|
||||||
|
|
||||||
|
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||||
|
plus sync DB writes — so it must NOT run on the event loop."""
|
||||||
|
return serialize_job(db, ingest_spooled(db, upload))
|
||||||
|
|
||||||
|
|
||||||
|
def ingest_spooled(db: Session, upload: SpooledUpload) -> DownloadJob:
|
||||||
|
"""The spooled upload into the library (`upload.ingest_upload`, which commits its own rows),
|
||||||
|
its refusals answered as HTTP."""
|
||||||
|
if upload.size == 0:
|
||||||
|
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
||||||
|
try:
|
||||||
|
job = uploadmod.ingest_upload(
|
||||||
|
db,
|
||||||
|
upload.user_id,
|
||||||
|
filename=upload.filename,
|
||||||
|
src_path=upload.path,
|
||||||
|
size=upload.size,
|
||||||
|
display_name=upload.display_name,
|
||||||
|
sha256=upload.sha256,
|
||||||
|
folder_id=upload.folder_id,
|
||||||
|
via=upload.via,
|
||||||
|
)
|
||||||
|
except quota.QuotaExceeded as e:
|
||||||
|
raise HTTPException(status_code=422, detail=quota_detail(e)) from e
|
||||||
|
except folders.FolderError as e: # the folder went while the file was on its way
|
||||||
|
raise refused(409, e.reason) from e
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_file(db: Session, job: DownloadJob) -> tuple[MediaAsset, Path]:
|
||||||
|
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||||
|
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||||
|
raise refused(409, "download_not_ready")
|
||||||
|
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||||
|
if path is None:
|
||||||
|
raise not_found("file")
|
||||||
|
return asset, path
|
||||||
|
|
||||||
|
|
||||||
|
def serve_file(db: Session, job: DownloadJob) -> FileResponse:
|
||||||
|
"""The file of `job` (the caller may read it), range-aware, named by the user's display name.
|
||||||
|
|
||||||
|
The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
|
||||||
|
(`storage.served_media_type`), NEVER guessed from the file's own extension. A generic 'file' (and
|
||||||
|
any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html")
|
||||||
|
returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on
|
||||||
|
this origin. Recognised media is served with its real type so the in-app player streams it. Both
|
||||||
|
are range-aware (FileResponse → 206). A file whose hash is known says so (`Repr-Digest`, RFC
|
||||||
|
9530: the digest of the whole file, whichever range is asked for)."""
|
||||||
|
asset, path = _ready_file(db, job)
|
||||||
|
filename = storage.download_filename(
|
||||||
|
job.display_name or asset.title or asset.source_ref, asset.container, path
|
||||||
|
)
|
||||||
|
headers = {}
|
||||||
|
if asset.sha256:
|
||||||
|
digest = base64.b64encode(bytes.fromhex(asset.sha256)).decode()
|
||||||
|
headers["Repr-Digest"] = f"sha-256=:{digest}:"
|
||||||
|
asset.last_access_at = datetime.now(timezone.utc) # the handler commits it
|
||||||
|
media_type = storage.served_media_type(asset.media_kind, asset.container)
|
||||||
|
if media_type is None:
|
||||||
|
return FileResponse(
|
||||||
|
path,
|
||||||
|
filename=filename,
|
||||||
|
media_type="application/octet-stream",
|
||||||
|
content_disposition_type="attachment",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
return FileResponse(path, filename=filename, media_type=media_type, headers=headers)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{job_id}/file")
|
||||||
|
def get_file(
|
||||||
|
job_id: int, caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
||||||
|
):
|
||||||
|
response = serve_file(db, readable_job(db, caller, job_id))
|
||||||
|
db.commit()
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
# A preview's own headers. `sandbox` leaves the answer no script and an opaque origin, whatever the
|
||||||
|
# file turns out to hold; `frame-ancestors 'self'` lets the app's viewer frame a PDF (every other
|
||||||
|
# answer keeps the app-wide 'none', main.py), with X-Frame-Options saying the same for old engines.
|
||||||
|
PREVIEW_HEADERS = {
|
||||||
|
"Content-Security-Policy": "sandbox; frame-ancestors 'self'",
|
||||||
|
"X-Frame-Options": "SAMEORIGIN",
|
||||||
|
}
|
||||||
|
# Set on a text preview cut at `preview.TEXT_PREVIEW_BYTES`, so the viewer can say it shows the start.
|
||||||
|
TRUNCATED_HEADER = "X-Preview-Truncated"
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{job_id}/preview")
|
||||||
|
def preview_file(
|
||||||
|
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||||
|
) -> Response:
|
||||||
|
"""The file shown in the library's viewer: an image or a PDF inline, a text format as plain text
|
||||||
|
(its first `TEXT_PREVIEW_BYTES`). The type comes from `downloads/preview.py`'s allow-list, so
|
||||||
|
an HTML or SVG upload arrives as text, never as a page. Anything else: 415 `file_no_preview`."""
|
||||||
|
job = accessible_job(db, user, job_id)
|
||||||
|
asset, path = _ready_file(db, job)
|
||||||
|
kind = preview.kind(asset.media_kind, asset.container)
|
||||||
|
if kind is None:
|
||||||
|
raise refused(415, "file_no_preview")
|
||||||
|
headers = dict(PREVIEW_HEADERS)
|
||||||
|
if kind == "text":
|
||||||
|
with path.open("rb") as f:
|
||||||
|
head = f.read(preview.TEXT_PREVIEW_BYTES + 1)
|
||||||
|
if len(head) > preview.TEXT_PREVIEW_BYTES:
|
||||||
|
head = head[: preview.TEXT_PREVIEW_BYTES]
|
||||||
|
headers[TRUNCATED_HEADER] = "1"
|
||||||
|
text = preview.text_utf8(head)
|
||||||
|
if text is None:
|
||||||
|
raise refused(415, "file_no_preview")
|
||||||
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
|
db.commit()
|
||||||
|
return Response(text, media_type=preview.media_type(kind, asset.container), headers=headers)
|
||||||
|
filename = storage.download_filename(
|
||||||
|
job.display_name or asset.title or asset.source_ref, asset.container, path
|
||||||
|
)
|
||||||
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
|
db.commit()
|
||||||
|
return FileResponse(
|
||||||
|
path,
|
||||||
|
filename=filename,
|
||||||
|
media_type=preview.media_type(kind, asset.container),
|
||||||
|
content_disposition_type="inline",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{job_id}/sha256")
|
||||||
|
def get_sha256(
|
||||||
|
job_id: int, caller: FilesCaller = Depends(files_caller), db: Session = Depends(get_db)
|
||||||
|
) -> dict:
|
||||||
|
"""The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder.
|
||||||
|
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool — and
|
||||||
|
every other sync route shares that pool, so only `_HASHERS` files are hashed at a time (a caller
|
||||||
|
past that is told to come back), and none of them holds a database connection while it reads."""
|
||||||
|
asset, path = _ready_file(db, readable_job(db, caller, job_id))
|
||||||
|
if asset.sha256:
|
||||||
|
return {"sha256": asset.sha256}
|
||||||
|
asset_id = asset.id
|
||||||
|
if not _HASHING.acquire(blocking=False):
|
||||||
|
raise refused(429, "file_hash_busy")
|
||||||
|
try:
|
||||||
|
release_read_connection(db)
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with path.open("rb") as f:
|
||||||
|
while chunk := f.read(_HASH_CHUNK):
|
||||||
|
h.update(chunk)
|
||||||
|
except OSError as e:
|
||||||
|
raise not_found("file") from e
|
||||||
|
finally:
|
||||||
|
_HASHING.release()
|
||||||
|
digest = h.hexdigest()
|
||||||
|
db.execute(
|
||||||
|
update(MediaAsset)
|
||||||
|
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
|
||||||
|
.values(sha256=digest)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return {"sha256": digest}
|
||||||
@@ -164,6 +164,7 @@ def unified_library(
|
|||||||
cast(null(), Integer).label("season_count"),
|
cast(null(), Integer).label("season_count"),
|
||||||
func.coalesce(st.position_seconds, 0).label("position_seconds"),
|
func.coalesce(st.position_seconds, 0).label("position_seconds"),
|
||||||
m_status.label("status"),
|
m_status.label("status"),
|
||||||
|
literal(False).label("resumable"),
|
||||||
# "last watched" sort key: when the movie was last resumed, else when it was marked
|
# "last watched" sort key: when the movie was last resumed, else when it was marked
|
||||||
# watched. NULL for never-touched movies → nullslast sinks them (see the sort dict).
|
# watched. NULL for never-touched movies → nullslast sinks them (see the sort dict).
|
||||||
func.coalesce(st.progress_updated_at, st.watched_at).label("last_watched"),
|
func.coalesce(st.progress_updated_at, st.watched_at).label("last_watched"),
|
||||||
@@ -211,6 +212,8 @@ def unified_library(
|
|||||||
PlexShow.child_count.label("season_count"),
|
PlexShow.child_count.label("season_count"),
|
||||||
literal(0).label("position_seconds"),
|
literal(0).label("position_seconds"),
|
||||||
s_status.label("status"),
|
s_status.label("status"),
|
||||||
|
# An episode holds a resume point (`ShowState.resumable`, the card's Continue + Restart).
|
||||||
|
(inp > 0).label("resumable"),
|
||||||
# Show leg's "last watched": max(resume/watched time) across the show's episodes (see
|
# Show leg's "last watched": max(resume/watched time) across the show's episodes (see
|
||||||
# rollup.agg_subquery include_resumed). NULL for a never-watched show → nullslast.
|
# rollup.agg_subquery include_resumed). NULL for a never-watched show → nullslast.
|
||||||
agg.c.resumed_at.label("last_watched"),
|
agg.c.resumed_at.label("last_watched"),
|
||||||
@@ -276,6 +279,7 @@ def unified_library(
|
|||||||
"thumb": f"/api/plex/image/{r.rk}",
|
"thumb": f"/api/plex/image/{r.rk}",
|
||||||
"season_count": r.season_count,
|
"season_count": r.season_count,
|
||||||
"status": r.status,
|
"status": r.status,
|
||||||
|
"resumable": r.resumable,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from app.models import (
|
|||||||
)
|
)
|
||||||
from app.plex import rollup
|
from app.plex import rollup
|
||||||
from app.plex import sync as plex_sync
|
from app.plex import sync as plex_sync
|
||||||
from app.plex.cards import display_title, movie_card, norm_lang, show_card
|
from app.plex.cards import SHOW_NEW, display_title, movie_card, norm_lang, show_card
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
from app.plex.query import content_libs, runtime_by_key, title_col, top_art_keys
|
from app.plex.query import content_libs, runtime_by_key, title_col, top_art_keys
|
||||||
@@ -176,6 +176,40 @@ def collections(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/collections/for/{item_rating_key}")
|
||||||
|
def collections_for_item(
|
||||||
|
item_rating_key: str,
|
||||||
|
user: User = Depends(current_user),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
"""What a movie or show card's "Add to collection" lists: the editable collections of the title's
|
||||||
|
library (a collection lives in one library, and only a manual one can take a title — the same
|
||||||
|
`can_edit` the editor offers), each saying whether the title is in it already, by name."""
|
||||||
|
movie = by_rating_key(db, PlexItem, item_rating_key)
|
||||||
|
title: PlexItem | PlexShow | None = (
|
||||||
|
movie if movie is not None and movie.kind == "movie" else None
|
||||||
|
) or by_rating_key(db, PlexShow, item_rating_key)
|
||||||
|
if title is None:
|
||||||
|
raise not_found("item")
|
||||||
|
member_of = set(title.collection_keys or [])
|
||||||
|
rows = (
|
||||||
|
db.execute(
|
||||||
|
select(PlexCollection)
|
||||||
|
.where(PlexCollection.library_id == title.library_id)
|
||||||
|
.order_by(collated(PlexCollection.title, lang_for_user(user)))
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"collections": [
|
||||||
|
{**_collection_card(c), "contains": c.rating_key in member_of}
|
||||||
|
for c in rows
|
||||||
|
if _collection_editable(c)
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class Member(NamedTuple):
|
class Member(NamedTuple):
|
||||||
"""One collection member flattened to everything the Sort control can order by. The page's two
|
"""One collection member flattened to everything the Sort control can order by. The page's two
|
||||||
legs are different tables (movies and shows), so there is no single query to ORDER BY — the
|
legs are different tables (movies and shows), so there is no single query to ORDER BY — the
|
||||||
@@ -284,7 +318,7 @@ def collection_detail(
|
|||||||
if shows
|
if shows
|
||||||
else []
|
else []
|
||||||
)
|
)
|
||||||
sstate = {r.show_id: rollup.show_status(r.total, r.watched, r.inprog) for r in show_rows}
|
sstate = {r.show_id: rollup.show_state(r.total, r.watched, r.inprog) for r in show_rows}
|
||||||
sresumed = {r.show_id: r.resumed_at for r in show_rows}
|
sresumed = {r.show_id: r.resumed_at for r in show_rows}
|
||||||
|
|
||||||
def _movie_last_watched(st: PlexState | None) -> datetime | None:
|
def _movie_last_watched(st: PlexState | None) -> datetime | None:
|
||||||
@@ -325,7 +359,7 @@ def collection_detail(
|
|||||||
items = [
|
items = [
|
||||||
movie_card(m.obj, mstate.get(m.obj.id), lang=lang)
|
movie_card(m.obj, mstate.get(m.obj.id), lang=lang)
|
||||||
if m.kind == "movie"
|
if m.kind == "movie"
|
||||||
else show_card(m.obj, sstate.get(m.obj.id, "new"), lang=lang)
|
else show_card(m.obj, sstate.get(m.obj.id, SHOW_NEW), lang=lang)
|
||||||
for m in members
|
for m in members
|
||||||
]
|
]
|
||||||
years = [m.year for m in members if m.year is not None]
|
years = [m.year for m in members if m.year is not None]
|
||||||
@@ -407,27 +441,47 @@ def create_collection(
|
|||||||
seed = body.item_rating_key
|
seed = body.item_rating_key
|
||||||
if not title or not seed:
|
if not title or not seed:
|
||||||
raise refused(400, "collection_fields_required")
|
raise refused(400, "collection_fields_required")
|
||||||
|
seed_show = by_rating_key(db, PlexShow, seed)
|
||||||
if library:
|
if library:
|
||||||
lib = (
|
lib = (
|
||||||
db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == library)).scalars().first()
|
db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == library)).scalars().first()
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# The "New collection" flow only knows the seed title → derive its library.
|
# The "New collection" flow only knows the seed title → derive its library.
|
||||||
seed_item = by_rating_key(db, PlexItem, seed) or by_rating_key(db, PlexShow, seed)
|
seed_item = by_rating_key(db, PlexItem, seed) or seed_show
|
||||||
lib = db.get(PlexLibrary, seed_item.library_id) if seed_item else None
|
lib = db.get(PlexLibrary, seed_item.library_id) if seed_item else None
|
||||||
if lib is None:
|
if lib is None:
|
||||||
raise not_found("library")
|
raise not_found("library")
|
||||||
_reject_duplicate_name(db, lib.id, title)
|
_reject_duplicate_name(db, lib.id, title)
|
||||||
wrote = False
|
wrote = False
|
||||||
|
new_rk = ""
|
||||||
try:
|
try:
|
||||||
with PlexClient(db) as plex:
|
with PlexClient(db) as plex:
|
||||||
new = plex.create_collection(lib.plex_key, title, seed)
|
# A show collection is made as one (Plex type 2): as a movie one it stayed empty.
|
||||||
|
new = plex.create_collection(lib.plex_key, title, seed, 2 if seed_show else 1)
|
||||||
wrote = True
|
wrote = True
|
||||||
col = plex_sync.resync_collection(db, plex, lib, str(new.get("ratingKey")))
|
new_rk = str(new.get("ratingKey") or "")
|
||||||
|
col = plex_sync.resync_collection(db, plex, lib, new_rk)
|
||||||
col.editable = True
|
col.editable = True
|
||||||
db.commit()
|
db.commit()
|
||||||
except (PlexError, PlexNotConfigured) as e:
|
except (PlexError, PlexNotConfigured) as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
|
if wrote and new_rk:
|
||||||
|
# It is on Plex now, and only the follow-up read failed. Its row is kept, editable: the
|
||||||
|
# full sync preserves `editable` but creates a missing row read-only, so the collection
|
||||||
|
# would drop out of every "add to" list; and the duplicate-name check reads these rows,
|
||||||
|
# so a retry of the same name would make a second one on Plex. Plex made it with exactly
|
||||||
|
# the seed in it, so the seed's row says so too — otherwise the title showed unticked in
|
||||||
|
# the collection it was just added to, and the collection empty, until that sync.
|
||||||
|
db.add(
|
||||||
|
PlexCollection(
|
||||||
|
rating_key=new_rk, library_id=lib.id, title=title, editable=True, child_count=1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||||
|
if row := by_rating_key(db, Model, seed):
|
||||||
|
row.collection_keys = list(dict.fromkeys([*(row.collection_keys or []), new_rk]))
|
||||||
|
db.commit()
|
||||||
raise _write_or_pending_error(wrote, e) from e
|
raise _write_or_pending_error(wrote, e) from e
|
||||||
return _collection_card(col)
|
return _collection_card(col)
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from app.collation import collated, sorted_words
|
|||||||
from app import logs
|
from app import logs
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
|
from app.discover import queries
|
||||||
from app.models import (
|
from app.models import (
|
||||||
PlexCollection,
|
PlexCollection,
|
||||||
PlexItem,
|
PlexItem,
|
||||||
@@ -28,7 +29,7 @@ from app.plex import people as plex_people
|
|||||||
from app.plex import rollup, subtitles
|
from app.plex import rollup, subtitles
|
||||||
from app.plex.query import title_col
|
from app.plex.query import title_col
|
||||||
from app.plex import tmdb as plex_tmdb
|
from app.plex import tmdb as plex_tmdb
|
||||||
from app.plex.cards import episode_card, movie_card, show_card
|
from app.plex.cards import SHOW_NEW, episode_card, movie_card, show_card
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
from app.routes.plex._shared import _norm_lang, item_or_404
|
from app.routes.plex._shared import _norm_lang, item_or_404
|
||||||
@@ -221,6 +222,24 @@ def show_detail(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/show/{rating_key}/next")
|
||||||
|
def show_next(
|
||||||
|
rating_key: str,
|
||||||
|
lang: str = "en",
|
||||||
|
user: User = Depends(current_user),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
"""The episode a show's card plays: the one in progress, else the first unwatched, else the first
|
||||||
|
(`discover.queries.next_episode`, the rule Home's shelves and the show page's Resume follow).
|
||||||
|
Asked one show at a time, when a card is hovered or played, rather than carried on every card of a
|
||||||
|
grid: three probes a show would cost a 60-card page 180 queries for the few cards anyone plays.
|
||||||
|
`next` is null for a show with no episodes."""
|
||||||
|
sh = by_rating_key(db, PlexShow, rating_key)
|
||||||
|
if sh is None:
|
||||||
|
raise not_found("show")
|
||||||
|
return {"next": queries.next_episode(db, user.id, sh.id, lang=_norm_lang(lang))}
|
||||||
|
|
||||||
|
|
||||||
def _season_block(se: PlexSeason, cards: list[dict]) -> dict:
|
def _season_block(se: PlexSeason, cards: list[dict]) -> dict:
|
||||||
"""A season's card for the show page (with its aggregate rollup) plus its episodes (used by the
|
"""A season's card for the show page (with its aggregate rollup) plus its episodes (used by the
|
||||||
season subpage, read from the same cached show-detail payload)."""
|
season subpage, read from the same cached show-detail payload)."""
|
||||||
@@ -260,7 +279,7 @@ def _relatedshow_cards(
|
|||||||
if sh is None or rk in seen:
|
if sh is None or rk in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(rk)
|
seen.add(rk)
|
||||||
out.append(show_card(sh, statuses.get(sh.id, "new"), lang=lang))
|
out.append(show_card(sh, statuses.get(sh.id, SHOW_NEW), lang=lang))
|
||||||
if len(out) >= 20:
|
if len(out) >= 20:
|
||||||
break
|
break
|
||||||
return out
|
return out
|
||||||
@@ -404,6 +423,10 @@ def item_detail(
|
|||||||
"type": m.get("type"),
|
"type": m.get("type"),
|
||||||
"start_s": round((m.get("startTimeOffset") or 0) / 1000, 1),
|
"start_s": round((m.get("startTimeOffset") or 0) / 1000, 1),
|
||||||
"end_s": round((m.get("endTimeOffset") or 0) / 1000, 1),
|
"end_s": round((m.get("endTimeOffset") or 0) / 1000, 1),
|
||||||
|
# An item can carry several credits blocks with scenes between them; Plex
|
||||||
|
# flags the one that really ends it. Which credits may leave the item is the
|
||||||
|
# player's rule (`lib/plexMarkers`), so the flag goes over as Plex sent it.
|
||||||
|
"final": bool(m.get("final")),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
# Audio + subtitle tracks (ordinals among their stream type — used to -map the selection).
|
# Audio + subtitle tracks (ordinals among their stream type — used to -map the selection).
|
||||||
@@ -500,6 +523,7 @@ def item_detail(
|
|||||||
|
|
||||||
prev_id, next_id = _episode_nav(db, it)
|
prev_id, next_id = _episode_nav(db, it)
|
||||||
show = db.get(PlexShow, it.show_id) if it.kind == "episode" and it.show_id else None
|
show = db.get(PlexShow, it.show_id) if it.kind == "episode" and it.show_id else None
|
||||||
|
season = db.get(PlexSeason, it.season_id) if show and it.season_id else None
|
||||||
return {
|
return {
|
||||||
"id": it.rating_key,
|
"id": it.rating_key,
|
||||||
"kind": it.kind,
|
"kind": it.kind,
|
||||||
@@ -533,6 +557,9 @@ def item_detail(
|
|||||||
"status": st.status if st else "new",
|
"status": st.status if st else "new",
|
||||||
"position_seconds": st.position_seconds if st else 0,
|
"position_seconds": st.position_seconds if st else 0,
|
||||||
"show_title": ((show.title_hu or show.title if hu else show.title) if show else None),
|
"show_title": ((show.title_hu or show.title if hu else show.title) if show else None),
|
||||||
|
# The episode page links up to its show and its season (their rating keys = the view keys).
|
||||||
|
"show_id": show.rating_key if show else None,
|
||||||
|
"season_id": season.rating_key if season else None,
|
||||||
"season_number": it.season_number,
|
"season_number": it.season_number,
|
||||||
"episode_number": it.episode_number,
|
"episode_number": it.episode_number,
|
||||||
"prev_id": prev_id,
|
"prev_id": prev_id,
|
||||||
|
|||||||
@@ -23,7 +23,13 @@ from app.auth import current_user
|
|||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
from app.models import PlexItem, PlexShow, PlexState, User
|
from app.models import PlexItem, PlexShow, PlexState, User
|
||||||
from app.plex import rollup
|
from app.plex import rollup
|
||||||
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
from app.plex.cards import (
|
||||||
|
SHOW_NEW,
|
||||||
|
display_title,
|
||||||
|
leaf_card,
|
||||||
|
movie_card,
|
||||||
|
show_card,
|
||||||
|
)
|
||||||
from app.plex.query import (
|
from app.plex.query import (
|
||||||
LibraryFilters,
|
LibraryFilters,
|
||||||
apply_meta_filters,
|
apply_meta_filters,
|
||||||
@@ -69,7 +75,7 @@ def _movie_cards(db: Session, uid: int, items: list[PlexItem], lang: str) -> lis
|
|||||||
|
|
||||||
def _show_cards(db: Session, uid: int, shows: list[PlexShow], lang: str) -> list[dict]:
|
def _show_cards(db: Session, uid: int, shows: list[PlexShow], lang: str) -> list[dict]:
|
||||||
statuses = rollup.state_map(db, uid, [s.id for s in shows])
|
statuses = rollup.state_map(db, uid, [s.id for s in shows])
|
||||||
return [show_card(s, statuses.get(s.id, "new"), lang=lang) for s in shows]
|
return [show_card(s, statuses.get(s.id, SHOW_NEW), lang=lang) for s in shows]
|
||||||
|
|
||||||
|
|
||||||
def _recently_added(
|
def _recently_added(
|
||||||
@@ -232,7 +238,9 @@ def _continue_watching(
|
|||||||
# `apply_show_state_filter` the grid uses, so the rail can't drift from the grid's definition.
|
# `apply_show_state_filter` the grid uses, so the rail can't drift from the grid's definition.
|
||||||
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
||||||
sq = apply_show_state_filter(
|
sq = apply_show_state_filter(
|
||||||
select(PlexShow, prog.c.resumed_at).join(prog, prog.c.show_id == PlexShow.id),
|
select(PlexShow, prog.c.resumed_at, prog.c.total, prog.c.watched, prog.c.inprog).join(
|
||||||
|
prog, prog.c.show_id == PlexShow.id
|
||||||
|
),
|
||||||
prog.c.total,
|
prog.c.total,
|
||||||
prog.c.watched,
|
prog.c.watched,
|
||||||
prog.c.inprog,
|
prog.c.inprog,
|
||||||
@@ -243,8 +251,9 @@ def _continue_watching(
|
|||||||
.order_by(prog.c.resumed_at.desc().nullslast())
|
.order_by(prog.c.resumed_at.desc().nullslast())
|
||||||
.limit(RAIL_LIMIT)
|
.limit(RAIL_LIMIT)
|
||||||
).all()
|
).all()
|
||||||
for sh, resumed_at in srows:
|
for sh, resumed_at, total, watched, inprog in srows:
|
||||||
rows.append((resumed_at, show_card(sh, "in_progress", lang=lang)))
|
state = rollup.show_state(total, watched, inprog)
|
||||||
|
rows.append((resumed_at, show_card(sh, state, lang=lang)))
|
||||||
|
|
||||||
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
||||||
return [card for _, card in rows[:RAIL_LIMIT]]
|
return [card for _, card in rows[:RAIL_LIMIT]]
|
||||||
@@ -402,11 +411,71 @@ def discover(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _draw(db: Session, stmt, key_col, per: int, exclude: list[str]) -> list:
|
||||||
|
"""`per` random rows of `stmt`, the previous draw's (`exclude`) left out while the pool allows: a
|
||||||
|
reshuffle over a small filtered pool kept dealing the same few titles. What the fresh rows leave
|
||||||
|
short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail."""
|
||||||
|
fresh = (
|
||||||
|
db.execute(stmt.where(key_col.not_in(exclude)).order_by(func.random()).limit(per))
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
if exclude
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
if len(fresh) == per:
|
||||||
|
return list(fresh)
|
||||||
|
taken = [key_col.not_in([r.rating_key for r in fresh])] if fresh else []
|
||||||
|
rest = db.execute(stmt.where(*taken).order_by(func.random()).limit(per - len(fresh))).scalars()
|
||||||
|
return [*fresh, *rest.all()]
|
||||||
|
|
||||||
|
|
||||||
|
def _kept(
|
||||||
|
db: Session, uid: int, movie_ids: list[int], show_ids: list[int], keep: list[str], lang: str
|
||||||
|
) -> list[dict]:
|
||||||
|
"""The `keep` titles' cards in their given order, as they stand now: a title watched since the draw
|
||||||
|
stays in it, marked watched, and one gone from the scope's libraries drops out."""
|
||||||
|
movies = (
|
||||||
|
db.execute(
|
||||||
|
select(PlexItem).where(
|
||||||
|
PlexItem.kind == "movie",
|
||||||
|
PlexItem.library_id.in_(movie_ids),
|
||||||
|
PlexItem.rating_key.in_(keep),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
if movie_ids
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
shows = (
|
||||||
|
db.execute(
|
||||||
|
select(PlexShow).where(PlexShow.library_id.in_(show_ids), PlexShow.rating_key.in_(keep))
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
if show_ids
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
cards = {
|
||||||
|
c["id"]: c
|
||||||
|
for c in [
|
||||||
|
*_movie_cards(db, uid, list(movies), lang),
|
||||||
|
*_show_cards(db, uid, list(shows), lang),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
return [cards[k] for k in keep if k in cards]
|
||||||
|
|
||||||
|
|
||||||
@router.get("/surprise")
|
@router.get("/surprise")
|
||||||
def surprise(
|
def surprise(
|
||||||
scope: str = "both",
|
scope: str = "both",
|
||||||
lang: str = "en",
|
lang: str = "en",
|
||||||
limit: int = Query(default=2 * SURPRISE_PER_KIND, ge=1, le=40),
|
limit: int = Query(default=2 * SURPRISE_PER_KIND, ge=1, le=40),
|
||||||
|
# The previous draw's rating keys: a reshuffle deals other titles first.
|
||||||
|
exclude: list[str] = Query(default=[], max_length=40),
|
||||||
|
# The draw on screen, read again rather than redrawn: its titles' watch state moves (a play, a
|
||||||
|
# toggle) while the draw itself stands until the ↻.
|
||||||
|
keep: list[str] = Query(default=[], max_length=40),
|
||||||
f: LibraryFilters = Depends(),
|
f: LibraryFilters = Depends(),
|
||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
@@ -419,6 +488,8 @@ def surprise(
|
|||||||
lang = _norm_lang(lang)
|
lang = _norm_lang(lang)
|
||||||
p = f.as_dict()
|
p = f.as_dict()
|
||||||
movie_ids, show_ids = _scope_libs(db, scope)
|
movie_ids, show_ids = _scope_libs(db, scope)
|
||||||
|
if keep:
|
||||||
|
return {"scope": scope, "items": _kept(db, user.id, movie_ids, show_ids, keep, lang)}
|
||||||
per = min(limit, SURPRISE_PER_KIND)
|
per = min(limit, SURPRISE_PER_KIND)
|
||||||
movies = []
|
movies = []
|
||||||
if movie_ids:
|
if movie_ids:
|
||||||
@@ -432,23 +503,21 @@ def surprise(
|
|||||||
)
|
)
|
||||||
.exists()
|
.exists()
|
||||||
)
|
)
|
||||||
movies = (
|
movies = _draw(
|
||||||
db.execute(
|
db,
|
||||||
_movie_meta(
|
_movie_meta(
|
||||||
select(PlexItem).where(
|
select(PlexItem).where(
|
||||||
PlexItem.kind == "movie",
|
PlexItem.kind == "movie",
|
||||||
PlexItem.library_id.in_(movie_ids),
|
PlexItem.library_id.in_(movie_ids),
|
||||||
~watched_movie,
|
~watched_movie,
|
||||||
),
|
),
|
||||||
p,
|
p,
|
||||||
lang,
|
lang,
|
||||||
f,
|
f,
|
||||||
)
|
),
|
||||||
.order_by(func.random())
|
PlexItem.rating_key,
|
||||||
.limit(per)
|
per,
|
||||||
)
|
exclude,
|
||||||
.scalars()
|
|
||||||
.all()
|
|
||||||
)
|
)
|
||||||
shows = []
|
shows = []
|
||||||
if show_ids:
|
if show_ids:
|
||||||
@@ -458,21 +527,19 @@ def surprise(
|
|||||||
not_fully_watched = or_(
|
not_fully_watched = or_(
|
||||||
prog.c.total.is_(None), prog.c.total == 0, prog.c.watched < prog.c.total
|
prog.c.total.is_(None), prog.c.total == 0, prog.c.watched < prog.c.total
|
||||||
)
|
)
|
||||||
shows = (
|
shows = _draw(
|
||||||
db.execute(
|
db,
|
||||||
apply_meta_filters(
|
apply_meta_filters(
|
||||||
select(PlexShow)
|
select(PlexShow)
|
||||||
.outerjoin(prog, prog.c.show_id == PlexShow.id)
|
.outerjoin(prog, prog.c.show_id == PlexShow.id)
|
||||||
.where(PlexShow.library_id.in_(show_ids), not_fully_watched),
|
.where(PlexShow.library_id.in_(show_ids), not_fully_watched),
|
||||||
PlexShow,
|
PlexShow,
|
||||||
p,
|
p,
|
||||||
lang,
|
lang,
|
||||||
)
|
),
|
||||||
.order_by(func.random())
|
PlexShow.rating_key,
|
||||||
.limit(per)
|
per,
|
||||||
)
|
exclude,
|
||||||
.scalars()
|
|
||||||
.all()
|
|
||||||
)
|
)
|
||||||
mcards = _movie_cards(db, user.id, movies, lang)
|
mcards = _movie_cards(db, user.id, movies, lang)
|
||||||
scards = _show_cards(db, user.id, shows, lang)
|
scards = _show_cards(db, user.id, shows, lang)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ from app.models import (
|
|||||||
)
|
)
|
||||||
from app.plex import rollup
|
from app.plex import rollup
|
||||||
from app.plex import tmdb as plex_tmdb
|
from app.plex import tmdb as plex_tmdb
|
||||||
from app.plex.cards import movie_card, norm_lang, show_card
|
from app.plex.cards import SHOW_NEW, movie_card, norm_lang, show_card
|
||||||
from app.plex.query import content_libs, split_csv, title_col
|
from app.plex.query import content_libs, split_csv, title_col
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||||
@@ -57,7 +57,7 @@ def _resolve_known_for(
|
|||||||
# A credit resolves against its OWN media_type's bucket only.
|
# A credit resolves against its OWN media_type's bucket only.
|
||||||
if k.get("media_type") == "tv":
|
if k.get("media_type") == "tv":
|
||||||
if g in shows:
|
if g in shows:
|
||||||
out.append(show_card(shows[g], show_statuses.get(shows[g].id, "new"), lang=lang))
|
out.append(show_card(shows[g], show_statuses.get(shows[g].id, SHOW_NEW), lang=lang))
|
||||||
elif g in movies:
|
elif g in movies:
|
||||||
out.append(movie_card(movies[g], m_states.get(movies[g].id), lang=lang))
|
out.append(movie_card(movies[g], m_states.get(movies[g].id), lang=lang))
|
||||||
return out[:_KNOWN_FOR_SHOWN]
|
return out[:_KNOWN_FOR_SHOWN]
|
||||||
@@ -245,7 +245,7 @@ def person_detail(
|
|||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
statuses = rollup.state_map(db, user.id, [s.id for s in show_rows])
|
statuses = rollup.state_map(db, user.id, [s.id for s in show_rows])
|
||||||
shows = [show_card(sh, statuses.get(sh.id, "new"), lang=lang) for sh in show_rows]
|
shows = [show_card(sh, statuses.get(sh.id, SHOW_NEW), lang=lang) for sh in show_rows]
|
||||||
|
|
||||||
known_for = _resolve_known_for(
|
known_for = _resolve_known_for(
|
||||||
db, person.known_for if person else None, user.id, movie_lib_ids, show_lib_ids, lang
|
db, person.known_for if person else None, user.id, movie_lib_ids, show_lib_ids, lang
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from fastapi import (
|
|||||||
Response,
|
Response,
|
||||||
)
|
)
|
||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse
|
||||||
from pydantic import BaseModel, ConfigDict, Field
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import select, update
|
from sqlalchemy import select, update
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
@@ -23,7 +23,6 @@ from sqlalchemy.orm import Session
|
|||||||
from app.auth import current_user, resolved_user_id
|
from app.auth import current_user, resolved_user_id
|
||||||
from app.db import SessionLocal, get_db
|
from app.db import SessionLocal, get_db
|
||||||
from app.models import (
|
from app.models import (
|
||||||
POSITION_MAX_S,
|
|
||||||
PlexItem,
|
PlexItem,
|
||||||
PlexSeason,
|
PlexSeason,
|
||||||
PlexShow,
|
PlexShow,
|
||||||
@@ -39,7 +38,7 @@ from app.plex import watch_sync as plex_watch
|
|||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
from app.routes.plex._shared import item_or_404
|
from app.routes.plex._shared import item_or_404
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
from app.routes._common import not_found
|
from app.routes._common import ProgressIn, not_found
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||||
|
|
||||||
@@ -160,11 +159,11 @@ def subtitle(
|
|||||||
return Response(subtitles.shift_vtt(hit, offset), media_type="text/vtt")
|
return Response(subtitles.shift_vtt(hit, offset), media_type="text/vtt")
|
||||||
|
|
||||||
|
|
||||||
# Resume/watched thresholds. _PROGRESS_MIN_S matches the feed's PROGRESS_MIN_SECONDS (5), but
|
# Resume/watched thresholds. _PROGRESS_MIN_S matches PROGRESS_MIN_SECONDS (5), but _FINISH_MARGIN_S is
|
||||||
# _FINISH_MARGIN_S is DELIBERATELY larger than the feed's FINISH_MARGIN_SECONDS (30 vs 10): this is
|
# DELIBERATELY larger than FINISH_MARGIN_SECONDS (30 vs 10): this is long-form content
|
||||||
# long-form content (movies/episodes with credits) and crossing it MARKS THE ITEM WATCHED, whereas
|
# (movies/episodes with credits) and crossing it MARKS THE ITEM WATCHED, whereas the feed's and the
|
||||||
# the feed's margin only clears a short video's resume bar — different domains, intentional divergence,
|
# library's margin only clears a resume bar — different domains, intentional divergence, not drift.
|
||||||
# not drift. Keep them separate on purpose (see routes/feed.py).
|
# Keep them separate on purpose (see routes/_common.py).
|
||||||
_PROGRESS_MIN_S = 5
|
_PROGRESS_MIN_S = 5
|
||||||
|
|
||||||
_FINISH_MARGIN_S = 30
|
_FINISH_MARGIN_S = 30
|
||||||
@@ -337,16 +336,9 @@ def _mark_finished(db: Session, user_id: int, item_id: int, now: datetime) -> bo
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
class ItemProgressIn(BaseModel):
|
class ItemProgressIn(ProgressIn):
|
||||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
"""The shared checkpoint (its bounds included), plus whether the player is leaving the item."""
|
||||||
# one into a 500.
|
|
||||||
model_config = ConfigDict(allow_inf_nan=False)
|
|
||||||
|
|
||||||
# float, truncated in the handler — the player may post a raw currentTime (a float); a plain
|
|
||||||
# `int` field would 422 it on the periodic progress checkpoint (the old `int(...)` truncated).
|
|
||||||
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
|
||||||
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
|
||||||
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
|
||||||
final: bool = False
|
final: bool = False
|
||||||
|
|
||||||
|
|
||||||
@@ -637,12 +629,27 @@ def stream_session(
|
|||||||
# The session id is part of the path (not a query param) so the playlist's RELATIVE segment
|
# The session id is part of the path (not a query param) so the playlist's RELATIVE segment
|
||||||
# names resolve back into the same session — see `stream_hls_file` for why it has to.
|
# names resolve back into the same session — see `stream_hls_file` for why it has to.
|
||||||
"url": f"/api/plex/stream/{it.rating_key}/hls/{s.id}/{s.entry}",
|
"url": f"/api/plex/stream/{it.rating_key}/hls/{s.id}/{s.entry}",
|
||||||
|
# What the player names when it no longer wants the session (POST …/session/{id}/stop).
|
||||||
|
"id": s.id,
|
||||||
"transcode": s.transcode,
|
"transcode": s.transcode,
|
||||||
"hw": s.hw,
|
"hw": s.hw,
|
||||||
"tonemap": s.tonemap,
|
"tonemap": s.tonemap,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/stream/{rating_key}/session/{session_id}/stop")
|
||||||
|
def stream_session_stop(
|
||||||
|
rating_key: str,
|
||||||
|
session_id: str,
|
||||||
|
user: User = Depends(current_user),
|
||||||
|
) -> dict:
|
||||||
|
"""The player no longer wants this session (it left the item, closed, or the page unloaded):
|
||||||
|
its ffmpeg stops now instead of at the idle reaper. A POST, so the player can send it as a
|
||||||
|
keepalive beacon from a page being unloaded. Stops nothing but the caller's own session, and
|
||||||
|
only while it is still the one registered for the item (see `stop_session`)."""
|
||||||
|
return {"ok": True, "stopped": plex_stream.stop_session(user.id, rating_key, session_id)}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/stream/{rating_key}/file")
|
@router.get("/stream/{rating_key}/file")
|
||||||
def stream_file(
|
def stream_file(
|
||||||
rating_key: str,
|
rating_key: str,
|
||||||
|
|||||||
+175
-56
@@ -9,14 +9,15 @@ a short-lived signed grant that the media URL carries (a `<video src>` can't sen
|
|||||||
from fastapi import APIRouter, Depends, Request
|
from fastapi import APIRouter, Depends, Request
|
||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select, update
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
from app.downloads import links as linksmod
|
from app.downloads import links as linksmod
|
||||||
from app.downloads import storage
|
from app.downloads import storage
|
||||||
from app.models import DownloadJob, DownloadLink, MediaAsset
|
from app.downloads.status import JobStatus
|
||||||
|
from app.models import DownloadJob, DownloadLink, FileFolder, MediaAsset, User
|
||||||
from app.ratelimit import RateLimiter, client_key
|
from app.ratelimit import RateLimiter, client_key
|
||||||
from app.security import verify_password
|
from app.security import verify_password
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
@@ -37,55 +38,154 @@ _unlock_failures = RateLimiter(max_events=50, window_seconds=300)
|
|||||||
|
|
||||||
|
|
||||||
def _link_or_404(db: Session, token: str) -> DownloadLink:
|
def _link_or_404(db: Session, token: str) -> DownloadLink:
|
||||||
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
|
link = linksmod.find(db, token)
|
||||||
if link is None or linksmod.is_expired(link):
|
if link is None or linksmod.is_expired(link):
|
||||||
raise refused(404, "link_gone")
|
raise refused(404, "link_gone")
|
||||||
return link
|
return link
|
||||||
|
|
||||||
|
|
||||||
def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAsset]:
|
def _being_made(db: Session, job: DownloadJob) -> bool:
|
||||||
job = db.get(DownloadJob, link.job_id)
|
"""The job is running, or queued for a worker that will take it: the worker leaves a suspended
|
||||||
|
owner's queued jobs where they are (`worker._claim_job`), as long as the suspension lasts."""
|
||||||
|
if job.status == JobStatus.RUNNING:
|
||||||
|
return True
|
||||||
|
if job.status != JobStatus.QUEUED:
|
||||||
|
return False
|
||||||
|
owner = db.get(User, job.user_id)
|
||||||
|
return owner is not None and not owner.is_suspended
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_job(db: Session, job: DownloadJob | None) -> tuple[DownloadJob, MediaAsset]:
|
||||||
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
||||||
|
if (
|
||||||
|
job is not None
|
||||||
|
and asset is not None
|
||||||
|
and asset.status in ("pending", "downloading")
|
||||||
|
and _being_made(db, job)
|
||||||
|
):
|
||||||
|
# Being (re)made — a re-download, or a conversion — so the link works again once it is ready:
|
||||||
|
# its own reason, so the page can say "try again shortly" rather than "this link is gone".
|
||||||
|
# Only while the job is still going: a cancelled or paused rebuild leaves the asset pending
|
||||||
|
# with nothing working on it, and "try again in a few minutes" would never come true.
|
||||||
|
raise refused(404, "video_preparing")
|
||||||
if job is None or asset is None or asset.status != "ready" or not asset.rel_path:
|
if job is None or asset is None or asset.status != "ready" or not asset.rel_path:
|
||||||
raise refused(404, "video_gone")
|
raise refused(404, "video_gone")
|
||||||
# Verify the file is actually on disk here (not just flagged ready), so the watch page never
|
# Verify the file is actually on disk here (not just flagged ready), so the watch page never
|
||||||
# loads a player for a missing file — meta and file agree.
|
# loads a player for a missing file — meta and file agree.
|
||||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||||
if path is None:
|
if path is None:
|
||||||
raise refused(404, "video_gone")
|
raise refused(404, "video_gone")
|
||||||
return job, asset
|
return job, asset
|
||||||
|
|
||||||
|
|
||||||
def _ready_asset(db: Session, link: DownloadLink) -> MediaAsset:
|
def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAsset]:
|
||||||
return _ready_target(db, link)[1]
|
"""A file link's file. A folder link names no one file: its files are asked for by id."""
|
||||||
|
if link.job_id is None:
|
||||||
|
raise refused(404, "video_gone")
|
||||||
|
return _ready_job(db, db.get(DownloadJob, link.job_id))
|
||||||
|
|
||||||
|
|
||||||
def _file_url(token: str, grant: str | None = None) -> str:
|
def _link_folder(db: Session, link: DownloadLink) -> FileFolder:
|
||||||
base = f"/api/public/watch/{token}/file"
|
folder = db.get(FileFolder, link.folder_id) if link.folder_id is not None else None
|
||||||
return f"{base}?g={grant}" if grant else base
|
if folder is None:
|
||||||
|
raise refused(404, "link_gone")
|
||||||
|
return folder
|
||||||
|
|
||||||
|
|
||||||
def _poster_url(token: str, asset, grant: str | None = None) -> str | None:
|
def _folder_job(db: Session, link: DownloadLink, job_id: int) -> tuple[DownloadJob, MediaAsset]:
|
||||||
|
folder = _link_folder(db, link)
|
||||||
|
job = db.execute(
|
||||||
|
select(DownloadJob).where(DownloadJob.id == job_id, *linksmod.in_folder(folder))
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if job is None:
|
||||||
|
raise refused(404, "video_gone")
|
||||||
|
return _ready_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
|
def _base(token: str, job_id: int | None) -> str:
|
||||||
|
"""A file link's file lives under its token; a folder link's files, each under its id."""
|
||||||
|
return f"/api/public/watch/{token}" + (f"/files/{job_id}" if job_id is not None else "")
|
||||||
|
|
||||||
|
|
||||||
|
def _file_url(token: str, grant: str | None = None, job_id: int | None = None) -> str:
|
||||||
|
url = f"{_base(token, job_id)}/file"
|
||||||
|
return f"{url}?g={grant}" if grant else url
|
||||||
|
|
||||||
|
|
||||||
|
def _poster_url(
|
||||||
|
token: str, asset: MediaAsset, grant: str | None = None, job_id: int | None = None
|
||||||
|
) -> str | None:
|
||||||
if not asset.poster_path:
|
if not asset.poster_path:
|
||||||
return None
|
return None
|
||||||
base = f"/api/public/watch/{token}/poster.jpg"
|
url = f"{_base(token, job_id)}/poster.jpg"
|
||||||
return f"{base}?g={grant}" if grant else base
|
return f"{url}?g={grant}" if grant else url
|
||||||
|
|
||||||
|
|
||||||
|
def _folder_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict:
|
||||||
|
"""A folder link's page: the folder's name and each file's own watch metadata
|
||||||
|
(`linksmod.folder_files`: ready, on disk, newest first)."""
|
||||||
|
folder = _link_folder(db, link)
|
||||||
|
files = [
|
||||||
|
{
|
||||||
|
"id": job.id,
|
||||||
|
**linksmod.public_meta(
|
||||||
|
link,
|
||||||
|
job,
|
||||||
|
asset,
|
||||||
|
_file_url(token, grant, job.id),
|
||||||
|
_poster_url(token, asset, grant, job.id),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
for job, asset in linksmod.folder_files(db, folder)
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"needs_password": False,
|
||||||
|
"kind": "folder",
|
||||||
|
"name": folder.name,
|
||||||
|
"allow_download": link.allow_download,
|
||||||
|
"files": files,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _file_meta(db: Session, link: DownloadLink, token: str, grant: str | None) -> dict:
|
||||||
|
job, asset = _ready_target(db, link)
|
||||||
|
return {
|
||||||
|
"needs_password": False,
|
||||||
|
"kind": "file",
|
||||||
|
**linksmod.public_meta(
|
||||||
|
link, job, asset, _file_url(token, grant), _poster_url(token, asset, grant)
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _opened(db: Session, link: DownloadLink, token: str, grant: str | None = None) -> dict:
|
||||||
|
"""The page an open link shows, counted as one view (the handler commits the count)."""
|
||||||
|
meta = (
|
||||||
|
_folder_meta(db, link, token, grant)
|
||||||
|
if link.folder_id is not None
|
||||||
|
else _file_meta(db, link, token, grant)
|
||||||
|
)
|
||||||
|
# In SQL, not `+= 1` on the loaded row: two opens at once would both write the same count.
|
||||||
|
db.execute(
|
||||||
|
update(DownloadLink)
|
||||||
|
.where(DownloadLink.id == link.id)
|
||||||
|
.values(view_count=DownloadLink.view_count + 1)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
)
|
||||||
|
return meta
|
||||||
|
|
||||||
|
|
||||||
@router.get("/watch/{token}")
|
@router.get("/watch/{token}")
|
||||||
def watch_meta(token: str, db: Session = Depends(get_db)) -> dict:
|
def watch_meta(token: str, db: Session = Depends(get_db)) -> dict:
|
||||||
"""Public metadata for the watch page. For a password link, returns only `needs_password`
|
"""Public metadata for the watch page: one file's, or a folder's list. For a password link,
|
||||||
until the viewer unlocks it (no title/thumbnail leak)."""
|
returns only `needs_password` until the viewer unlocks it (no title/thumbnail leak)."""
|
||||||
link = _link_or_404(db, token)
|
link = _link_or_404(db, token)
|
||||||
if link.password_hash:
|
if link.password_hash:
|
||||||
return {"needs_password": True}
|
# Which kind it opens, for the password prompt's title — never what is in it.
|
||||||
job, asset = _ready_target(db, link)
|
return {"needs_password": True, "kind": "folder" if link.folder_id is not None else "file"}
|
||||||
link.view_count += 1
|
meta = _opened(db, link, token)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {
|
return meta
|
||||||
"needs_password": False,
|
|
||||||
**linksmod.public_meta(link, job, asset, _file_url(token), _poster_url(token, asset)),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class WatchUnlockIn(BaseModel):
|
class WatchUnlockIn(BaseModel):
|
||||||
@@ -96,7 +196,7 @@ class WatchUnlockIn(BaseModel):
|
|||||||
def watch_unlock(
|
def watch_unlock(
|
||||||
token: str, body: WatchUnlockIn, request: Request, db: Session = Depends(get_db)
|
token: str, body: WatchUnlockIn, request: Request, db: Session = Depends(get_db)
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Exchange the link password for a signed grant carried by the media URL."""
|
"""Exchange the link password for a signed grant carried by the media URLs."""
|
||||||
# The link first: a token naming none must leave no limiter entry behind, or posting made-up tokens
|
# The link first: a token naming none must leave no limiter entry behind, or posting made-up tokens
|
||||||
# would grow the limiter without bound.
|
# would grow the limiter without bound.
|
||||||
link = _link_or_404(db, token)
|
link = _link_or_404(db, token)
|
||||||
@@ -105,37 +205,28 @@ def watch_unlock(
|
|||||||
raise refused(429, "too_many_attempts")
|
raise refused(429, "too_many_attempts")
|
||||||
if not link.password_hash:
|
if not link.password_hash:
|
||||||
# Not a password link — nothing to unlock; just hand back the plain meta.
|
# Not a password link — nothing to unlock; just hand back the plain meta.
|
||||||
job, asset = _ready_target(db, link)
|
meta = _opened(db, link, token)
|
||||||
return {
|
db.commit()
|
||||||
"ok": True,
|
return {"ok": True, **meta}
|
||||||
"needs_password": False,
|
|
||||||
**linksmod.public_meta(link, job, asset, _file_url(token), _poster_url(token, asset)),
|
|
||||||
}
|
|
||||||
if not verify_password(body.password, link.password_hash):
|
if not verify_password(body.password, link.password_hash):
|
||||||
_unlock_failures.allow(token)
|
_unlock_failures.allow(token)
|
||||||
raise refused(403, "wrong_share_password")
|
raise refused(403, "wrong_share_password")
|
||||||
job, asset = _ready_target(db, link)
|
|
||||||
link.view_count += 1
|
|
||||||
db.commit()
|
|
||||||
grant = linksmod.make_grant(token, link.password_version)
|
grant = linksmod.make_grant(token, link.password_version)
|
||||||
return {
|
meta = _opened(db, link, token, grant)
|
||||||
"ok": True,
|
db.commit()
|
||||||
"needs_password": False,
|
return {"ok": True, **meta}
|
||||||
**linksmod.public_meta(
|
|
||||||
link, job, asset, _file_url(token, grant), _poster_url(token, asset, grant)
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/watch/{token}/file")
|
def _granted(link: DownloadLink, token: str, g: str | None) -> None:
|
||||||
def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
"""A password link's files want the grant its unlock handed out."""
|
||||||
"""Range-aware stream of the shared file. Inline by default (plays in the page); an
|
|
||||||
`allow_download` link serves it as an attachment. Password links require a valid grant."""
|
|
||||||
link = _link_or_404(db, token)
|
|
||||||
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
||||||
raise refused(403, "share_password_required")
|
raise refused(403, "share_password_required")
|
||||||
asset = _ready_asset(db, link)
|
|
||||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
|
||||||
|
def _serve(link: DownloadLink, asset: MediaAsset) -> FileResponse:
|
||||||
|
"""Range-aware stream of a shared file. Inline by default (plays in the page); an
|
||||||
|
`allow_download` link serves it as an attachment."""
|
||||||
|
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||||
if path is None:
|
if path is None:
|
||||||
raise refused(404, "video_gone")
|
raise refused(404, "video_gone")
|
||||||
|
|
||||||
@@ -161,17 +252,45 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _poster(asset: MediaAsset) -> FileResponse:
|
||||||
|
if not asset.poster_path:
|
||||||
|
raise not_found("poster")
|
||||||
|
path = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||||
|
if path is None:
|
||||||
|
raise not_found("poster")
|
||||||
|
return FileResponse(path, media_type="image/jpeg")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/watch/{token}/file")
|
||||||
|
def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
||||||
|
"""A file link's file. Password links require a valid grant."""
|
||||||
|
link = _link_or_404(db, token)
|
||||||
|
_granted(link, token, g)
|
||||||
|
return _serve(link, _ready_target(db, link)[1])
|
||||||
|
|
||||||
|
|
||||||
@router.get("/watch/{token}/poster.jpg")
|
@router.get("/watch/{token}/poster.jpg")
|
||||||
def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
||||||
"""The generated poster frame for a shared thumbnail-less video (used as the page/video poster
|
"""The generated poster frame for a shared thumbnail-less video (used as the page/video poster
|
||||||
and the link-preview image). Password links require a valid grant, like the file."""
|
and the link-preview image). Password links require a valid grant, like the file."""
|
||||||
link = _link_or_404(db, token)
|
link = _link_or_404(db, token)
|
||||||
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
_granted(link, token, g)
|
||||||
raise refused(403, "share_password_required")
|
return _poster(_ready_target(db, link)[1])
|
||||||
job, asset = _ready_target(db, link)
|
|
||||||
if not asset.poster_path:
|
|
||||||
raise not_found("poster")
|
@router.get("/watch/{token}/files/{job_id}/file")
|
||||||
path = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
def watch_folder_file(token: str, job_id: int, g: str | None = None, db: Session = Depends(get_db)):
|
||||||
if path is None:
|
"""One file of a folder link's folder, while it is in it. Password links require a grant."""
|
||||||
raise not_found("poster")
|
link = _link_or_404(db, token)
|
||||||
return FileResponse(path, media_type="image/jpeg")
|
_granted(link, token, g)
|
||||||
|
return _serve(link, _folder_job(db, link, job_id)[1])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/watch/{token}/files/{job_id}/poster.jpg")
|
||||||
|
def watch_folder_poster(
|
||||||
|
token: str, job_id: int, g: str | None = None, db: Session = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""The poster frame of one file of a folder link's folder, under the same terms as the file."""
|
||||||
|
link = _link_or_404(db, token)
|
||||||
|
_granted(link, token, g)
|
||||||
|
return _poster(_folder_job(db, link, job_id)[1])
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
"""A user's API tokens for their Files library (`app.apitokens`): list, make, revoke.
|
||||||
|
|
||||||
|
Signed-in only — `current_user`, never a token: a token cannot make itself a wider one. A new
|
||||||
|
token's value is in the answer that made it and nowhere after; only its hash is kept.
|
||||||
|
|
||||||
|
The Hub preset is the one machine a user swaps files with: it makes (or reuses) their Hub folder
|
||||||
|
and gives the token every right in it, so the hub both drops files there and picks up the ones put
|
||||||
|
there for it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import apitokens, audit
|
||||||
|
from app.apitokens import GRANT_COLUMNS, Right
|
||||||
|
from app.audit import AuditAction
|
||||||
|
from app.auth import current_user
|
||||||
|
from app.db import get_db
|
||||||
|
from app.downloads import folders
|
||||||
|
from app.models import ApiToken, ApiTokenGrant, FileFolder, User
|
||||||
|
from app.refusal import refused
|
||||||
|
from app.routes._common import owned_or_404
|
||||||
|
from app.security import hash_token
|
||||||
|
from app.utils import iso
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/tokens", tags=["tokens"])
|
||||||
|
|
||||||
|
# Far more than one person's machines; a ceiling, so a script gone wrong cannot mint without end.
|
||||||
|
MAX_TOKENS = 20
|
||||||
|
_MAX_GRANTS = 50
|
||||||
|
_MAX_EXPIRY_DAYS = 3650
|
||||||
|
|
||||||
|
|
||||||
|
class GrantIn(BaseModel):
|
||||||
|
folder_id: int
|
||||||
|
rights: list[Right] = Field(min_length=1, max_length=len(Right))
|
||||||
|
|
||||||
|
|
||||||
|
class TokenIn(BaseModel):
|
||||||
|
name: str = Field(min_length=1, max_length=80)
|
||||||
|
# None = never expires.
|
||||||
|
expires_days: int | None = Field(default=None, ge=1, le=_MAX_EXPIRY_DAYS)
|
||||||
|
preset: Literal["hub"] | None = None
|
||||||
|
grants: list[GrantIn] = Field(default_factory=list, max_length=_MAX_GRANTS)
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize(db: Session, token: ApiToken) -> dict:
|
||||||
|
rows = db.execute(
|
||||||
|
select(ApiTokenGrant, FileFolder)
|
||||||
|
.join(FileFolder, FileFolder.id == ApiTokenGrant.folder_id)
|
||||||
|
.where(ApiTokenGrant.token_id == token.id)
|
||||||
|
.order_by(FileFolder.id)
|
||||||
|
).all()
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
return {
|
||||||
|
"id": token.id,
|
||||||
|
"name": token.name,
|
||||||
|
"prefix": token.prefix,
|
||||||
|
"created_at": iso(token.created_at),
|
||||||
|
"expires_at": iso(token.expires_at),
|
||||||
|
"expired": token.expires_at is not None and token.expires_at <= now,
|
||||||
|
"expiring": apitokens.expiring(token, now),
|
||||||
|
"last_used_at": iso(token.last_used_at),
|
||||||
|
"grants": [
|
||||||
|
{
|
||||||
|
"folder_id": folder.id,
|
||||||
|
"kind": folder.kind,
|
||||||
|
"source_key": folder.source_key,
|
||||||
|
"name": folder.name,
|
||||||
|
"rights": sorted(apitokens.grant_rights(grant)),
|
||||||
|
}
|
||||||
|
for grant, folder in rows
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("")
|
||||||
|
def list_tokens(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||||
|
tokens = (
|
||||||
|
db.execute(select(ApiToken).where(ApiToken.user_id == user.id).order_by(ApiToken.id))
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return [_serialize(db, t) for t in tokens]
|
||||||
|
|
||||||
|
|
||||||
|
def _wanted_grants(db: Session, user: User, body: TokenIn) -> dict[int, set[Right]]:
|
||||||
|
"""Folder id → the rights the new token gets there; every folder checked to be the user's, and
|
||||||
|
writing only where an upload may land (`folders.upload_target`)."""
|
||||||
|
if body.preset == "hub":
|
||||||
|
if body.grants:
|
||||||
|
raise refused(400, "api_token_preset_grants")
|
||||||
|
return {folders.hub_folder(db, user.id).id: set(Right)}
|
||||||
|
if not body.grants:
|
||||||
|
raise refused(400, "api_token_no_grants")
|
||||||
|
wanted: dict[int, set[Right]] = {}
|
||||||
|
for grant in body.grants:
|
||||||
|
wanted.setdefault(grant.folder_id, set()).update(grant.rights)
|
||||||
|
for folder_id, rights in wanted.items():
|
||||||
|
try:
|
||||||
|
if Right.WRITE in rights:
|
||||||
|
folders.upload_target(db, user.id, folder_id)
|
||||||
|
else:
|
||||||
|
folders.own_folder(db, user.id, folder_id)
|
||||||
|
except folders.FolderError as e:
|
||||||
|
raise refused(409, e.reason) from e
|
||||||
|
return wanted
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("")
|
||||||
|
def create_token(
|
||||||
|
body: TokenIn, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||||
|
) -> dict:
|
||||||
|
"""Make a token; its value is in this answer only."""
|
||||||
|
name = body.name.strip()
|
||||||
|
if not name:
|
||||||
|
raise refused(400, "api_token_name_required")
|
||||||
|
# The user's row, locked: two requests at once cannot both slip under the ceiling, and a
|
||||||
|
# session ended since `current_user` read the epoch (unlocked) cannot mint a token that outlives
|
||||||
|
# it — `bump_session_epoch` takes this lock before it revokes them all. No key update: a row that
|
||||||
|
# only refers to the user (a folder delete's audit line) must not wait on it, or the two deadlock.
|
||||||
|
epoch = db.execute(
|
||||||
|
select(User.session_epoch).where(User.id == user.id).with_for_update(key_share=True)
|
||||||
|
).scalar_one()
|
||||||
|
if (epoch or 0) != (user.session_epoch or 0):
|
||||||
|
raise refused(401, "not_authenticated")
|
||||||
|
count = db.execute(
|
||||||
|
select(func.count(ApiToken.id)).where(ApiToken.user_id == user.id)
|
||||||
|
).scalar_one()
|
||||||
|
if count >= MAX_TOKENS:
|
||||||
|
raise refused(409, "api_token_limit", limit=MAX_TOKENS)
|
||||||
|
wanted = _wanted_grants(db, user, body)
|
||||||
|
raw = apitokens.mint()
|
||||||
|
token = ApiToken(
|
||||||
|
user_id=user.id,
|
||||||
|
name=name,
|
||||||
|
prefix=raw[: apitokens.SHOWN_LEN],
|
||||||
|
token_hash=hash_token(raw),
|
||||||
|
expires_at=None
|
||||||
|
if body.expires_days is None
|
||||||
|
else datetime.now(timezone.utc) + timedelta(days=body.expires_days),
|
||||||
|
)
|
||||||
|
db.add(token)
|
||||||
|
db.flush()
|
||||||
|
for folder_id, rights in wanted.items():
|
||||||
|
db.add(
|
||||||
|
ApiTokenGrant(
|
||||||
|
token_id=token.id,
|
||||||
|
folder_id=folder_id,
|
||||||
|
**{col: right in rights for right, col in GRANT_COLUMNS.items()},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.flush()
|
||||||
|
audit.record(
|
||||||
|
db,
|
||||||
|
user.id,
|
||||||
|
AuditAction.API_TOKEN_CREATE,
|
||||||
|
target_type="api_token",
|
||||||
|
target_id=token.id,
|
||||||
|
after={
|
||||||
|
"grants": {str(f): sorted(r) for f, r in wanted.items()},
|
||||||
|
"expires_at": iso(token.expires_at),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return {"ok": True, **_serialize(db, token), "token": raw}
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{token_id}")
|
||||||
|
def revoke_token(
|
||||||
|
token_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||||
|
) -> dict:
|
||||||
|
"""Revoke a token: it stops working at its next request (its grants go with it)."""
|
||||||
|
token = owned_or_404(db, ApiToken, token_id, user, what="api_token")
|
||||||
|
db.delete(token)
|
||||||
|
audit.record(
|
||||||
|
db, user.id, AuditAction.API_TOKEN_REVOKE, target_type="api_token", target_id=token_id
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return {"ok": True, "deleted": token_id}
|
||||||
@@ -370,7 +370,8 @@ def _explore_cleanup_job() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _download_gc_job() -> None:
|
def _download_gc_job() -> None:
|
||||||
# Retention GC for the download center: pre-expiry warnings, TTL deletion, LRU eviction.
|
# Retention GC for the download center: pre-expiry warnings, TTL deletion, LRU eviction — and
|
||||||
|
# the week-ahead warning before an API token runs out (`apitokens.warn_expiring`).
|
||||||
# Pure disk/DB work (no YouTube quota).
|
# Pure disk/DB work (no YouTube quota).
|
||||||
_job("download_gc", run_download_gc)
|
_job("download_gc", run_download_gc)
|
||||||
|
|
||||||
|
|||||||
@@ -46,6 +46,12 @@ _fernet: Fernet | None = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def can_encrypt() -> bool:
|
||||||
|
"""Whether a value can be stored encrypted here: TOKEN_ENCRYPTION_KEY is set. Production
|
||||||
|
refuses to start without it (`config`); a local box may run without one."""
|
||||||
|
return _fernet is not None
|
||||||
|
|
||||||
|
|
||||||
def _require_fernet() -> Fernet:
|
def _require_fernet() -> Fernet:
|
||||||
if _fernet is None:
|
if _fernet is None:
|
||||||
raise RuntimeError("TOKEN_ENCRYPTION_KEY is not configured")
|
raise RuntimeError("TOKEN_ENCRYPTION_KEY is not configured")
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ result (measured on real data: every untagged channel got a suggestion, most on
|
|||||||
alone). Topic overlap is what makes "you tagged similar channels this way" meaningful.
|
alone). Topic overlap is what makes "you tagged similar channels this way" meaningful.
|
||||||
|
|
||||||
Purely read-only and deterministic — the Channel Manager surfaces the result as one-tap-accept
|
Purely read-only and deterministic — the Channel Manager surfaces the result as one-tap-accept
|
||||||
"ghost" chips; nothing is written here (D4 = suggest-and-confirm, no silent auto-apply). The
|
"ghost" chips; nothing is written here (suggest-and-confirm, never a silent auto-apply). The
|
||||||
thresholds are tuning constants rather than admin knobs on purpose: they mirror how ``autotag.py``
|
thresholds are tuning constants rather than admin knobs on purpose: they mirror how ``autotag.py``
|
||||||
keeps its mappings inline, and sysconfig only stores int/str/bool (no float), so a knob would mean
|
keeps its mappings inline, and sysconfig only stores int/str/bool (no float), so a knob would mean
|
||||||
storing percentages — not worth the surface for values an operator has no reason to tune per-install.
|
storing percentages — not worth the surface for values an operator has no reason to tune per-install.
|
||||||
|
|||||||
+47
-13
@@ -29,6 +29,7 @@ from datetime import datetime, timezone
|
|||||||
from functools import partial
|
from functools import partial
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs
|
from app import logs
|
||||||
from app import logsink
|
from app import logsink
|
||||||
@@ -36,7 +37,7 @@ from app import sysconfig
|
|||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.downloads import credentials, formats, quota, service
|
from app.downloads import credentials, formats, quota, service, storage
|
||||||
from app.downloads import errors as dlerrors
|
from app.downloads import errors as dlerrors
|
||||||
from app.downloads.jobdb import (
|
from app.downloads.jobdb import (
|
||||||
_edit_staging_dir,
|
_edit_staging_dir,
|
||||||
@@ -44,7 +45,7 @@ from app.downloads.jobdb import (
|
|||||||
_reset_asset_pending,
|
_reset_asset_pending,
|
||||||
_staging_dir,
|
_staging_dir,
|
||||||
)
|
)
|
||||||
from app.downloads.status import STOPPED_BY_ROUTE, JobStatus
|
from app.downloads.status import STOPPED_BY_ROUTE, JobKind, JobStatus
|
||||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
@@ -162,7 +163,12 @@ def _claim_job() -> int | None:
|
|||||||
seen.) The per-user limit is the quota row's `max_concurrent` (else the sysconfig default);
|
seen.) The per-user limit is the quota row's `max_concurrent` (else the sysconfig default);
|
||||||
`unlimited` users are never capped. A final Python `at_concurrency_limit` re-check still guards
|
`unlimited` users are never capped. A final Python `at_concurrency_limit` re-check still guards
|
||||||
the tiny race between the exclusion subquery and the claim (it can only ever reject, never
|
the tiny race between the exclusion subquery and the claim (it can only ever reject, never
|
||||||
over-claim), so correctness never depends on the two observations agreeing."""
|
over-claim), so correctness never depends on the two observations agreeing.
|
||||||
|
|
||||||
|
A suspended user's queued jobs are left where they are: suspension blocks their downloads (not
|
||||||
|
only their sign-in), and lifting it lets them run in their old place. A job already running
|
||||||
|
when the user is suspended finishes on that worker; one a worker restart puts back in the queue
|
||||||
|
(`_requeue_if_running`, `_recover_orphans`) waits with the rest."""
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
default_mc = sysconfig.get_int(db, "download_default_max_concurrent")
|
default_mc = sysconfig.get_int(db, "download_default_max_concurrent")
|
||||||
rows = db.execute(
|
rows = db.execute(
|
||||||
@@ -170,6 +176,10 @@ def _claim_job() -> int | None:
|
|||||||
"""
|
"""
|
||||||
SELECT id, user_id FROM download_jobs
|
SELECT id, user_id FROM download_jobs
|
||||||
WHERE status='queued'
|
WHERE status='queued'
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM users u
|
||||||
|
WHERE u.id = download_jobs.user_id AND u.is_suspended
|
||||||
|
)
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1 FROM media_assets a
|
SELECT 1 FROM media_assets a
|
||||||
WHERE a.id = download_jobs.asset_id AND a.status = 'downloading'
|
WHERE a.id = download_jobs.asset_id AND a.status = 'downloading'
|
||||||
@@ -437,7 +447,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
|||||||
if status is None or status in STOPPED_BY_ROUTE:
|
if status is None or status in STOPPED_BY_ROUTE:
|
||||||
reason = "cancelled" # a route stopped it (or the row vanished)
|
reason = "cancelled" # a route stopped it (or the row vanished)
|
||||||
break
|
break
|
||||||
if status == "running":
|
if status == JobStatus.RUNNING:
|
||||||
if stale is not None and stale > stall_s:
|
if stale is not None and stale > stall_s:
|
||||||
reason = "stalled"
|
reason = "stalled"
|
||||||
break
|
break
|
||||||
@@ -448,7 +458,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
|||||||
# teardown. A still-running job over the cap is failed ("timeout"); a terminal one (already
|
# teardown. A still-running job over the cap is failed ("timeout"); a terminal one (already
|
||||||
# done/error) just gets its lingering child killed, its DB status left intact ("abandon").
|
# done/error) just gets its lingering child killed, its DB status left intact ("abandon").
|
||||||
if time.monotonic() - started > wall_s:
|
if time.monotonic() - started > wall_s:
|
||||||
reason = "timeout" if status == "running" else "abandon"
|
reason = "timeout" if status == JobStatus.RUNNING else "abandon"
|
||||||
break
|
break
|
||||||
# else: a terminal status under the cap (done/error, or a re-queued contended asset) means
|
# else: a terminal status under the cap (done/error, or a re-queued contended asset) means
|
||||||
# the child is exiting on its own — keep polling; NEVER kill a finishing child (that could
|
# the child is exiting on its own — keep polling; NEVER kill a finishing child (that could
|
||||||
@@ -479,7 +489,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
|||||||
_reset_asset_pending(asset_id)
|
_reset_asset_pending(asset_id)
|
||||||
if proc.returncode != _CHILD_OK:
|
if proc.returncode != _CHILD_OK:
|
||||||
status, _ = _job_liveness(job_id)
|
status, _ = _job_liveness(job_id)
|
||||||
if status == "running":
|
if status == JobStatus.RUNNING:
|
||||||
_cleanup_staging(asset_id)
|
_cleanup_staging(asset_id)
|
||||||
_fail_running(
|
_fail_running(
|
||||||
job_id,
|
job_id,
|
||||||
@@ -560,15 +570,12 @@ def _process_job(job_id: int) -> None:
|
|||||||
asset_id = asset.id if asset else None
|
asset_id = asset.id if asset else None
|
||||||
asset_status = asset.status if asset else None
|
asset_status = asset.status if asset else None
|
||||||
asset_error = asset.error if asset else None
|
asset_error = asset.error if asset else None
|
||||||
if asset_status == "error":
|
if asset is not None and asset_status == "error":
|
||||||
# A job that meets an already-failed asset gets that failure's explanation — not a bare
|
# A job that meets an already-failed asset gets that failure's explanation — not a bare
|
||||||
# "Download failed". Only the text is on the asset, so it is diagnosed again, for this
|
# "Download failed". Only the text is on the asset, so it is diagnosed again, for this
|
||||||
# link, with whether a cookie goes with it asked of the same gate the download child
|
# link, with whether a cookie goes with it asked of the same gate the download child
|
||||||
# used (`credentials.usable_cookie`) — as it stands now, so the advice fits this job's Retry.
|
# used (`credentials.usable_cookie`) — as it stands now, so the advice fits this job's Retry.
|
||||||
url = service.source_url(job.source_kind, job.source_ref)
|
url, cookie = _failed_link_and_cookie(db, job, asset)
|
||||||
cookie = credentials.usable_cookie(
|
|
||||||
db, job.user_id, url, asset, formats.normalize(job.profile_snapshot)
|
|
||||||
)
|
|
||||||
diagnosis = dlerrors.diagnose(asset_error, url=url, cookie_attached=cookie is not None)
|
diagnosis = dlerrors.diagnose(asset_error, url=url, cookie_attached=cookie is not None)
|
||||||
del cookie # only whether there is one matters here
|
del cookie # only whether there is one matters here
|
||||||
|
|
||||||
@@ -605,13 +612,34 @@ def _process_job(job_id: int) -> None:
|
|||||||
_run_supervised(job_id, asset_id)
|
_run_supervised(job_id, asset_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _failed_link_and_cookie(
|
||||||
|
db: Session, job: DownloadJob, asset: MediaAsset
|
||||||
|
) -> tuple[str | None, str | None]:
|
||||||
|
"""The link a job's failed asset was fetched from, and the cookie that fetch would go with now.
|
||||||
|
A clip's link is its source's (a `clip` ref names the user's range, no page), asked of the
|
||||||
|
clip's own cookie gate, as its fetch was (`_clip_source`); a Plex clip fetches nothing."""
|
||||||
|
if job.job_kind == JobKind.CLIP:
|
||||||
|
spec = dict(job.edit_spec or {})
|
||||||
|
if spec.get("source") != "youtube":
|
||||||
|
return None, None
|
||||||
|
url = service.source_url("youtube", spec["ref"])
|
||||||
|
return url, credentials.usable_clip_cookie(db, job.user_id, url, asset, spec)
|
||||||
|
url = service.source_url(job.source_kind, job.source_ref)
|
||||||
|
spec = formats.normalize(job.profile_snapshot)
|
||||||
|
return url, credentials.usable_cookie(db, job.user_id, url, asset, spec)
|
||||||
|
|
||||||
|
|
||||||
def _finish_from_cache(job_id: int, asset_id: int) -> None:
|
def _finish_from_cache(job_id: int, asset_id: int) -> None:
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
job = db.get(DownloadJob, job_id)
|
job = db.get(DownloadJob, job_id)
|
||||||
asset = db.get(MediaAsset, asset_id)
|
asset = db.get(MediaAsset, asset_id)
|
||||||
if job and asset:
|
if job and asset:
|
||||||
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
||||||
if not job.display_name and asset.title:
|
# A clip queued under the placeholder takes the title its sibling's fetch found.
|
||||||
|
unnamed = not job.display_name or (
|
||||||
|
job.job_kind == JobKind.CLIP and job.display_name == service.clip_title(None, None)
|
||||||
|
)
|
||||||
|
if unnamed and asset.title:
|
||||||
# display_name is VARCHAR(255); a title can be longer (e.g. a recipe in the title).
|
# display_name is VARCHAR(255); a title can be longer (e.g. a recipe in the title).
|
||||||
fields["display_name"] = asset.title[:DISPLAY_TEXT_MAX]
|
fields["display_name"] = asset.title[:DISPLAY_TEXT_MAX]
|
||||||
asset.last_access_at = datetime.now(timezone.utc)
|
asset.last_access_at = datetime.now(timezone.utc)
|
||||||
@@ -682,12 +710,18 @@ def main() -> None:
|
|||||||
# `log_events`, so on the deploy that first adds the table the worker may start before the API
|
# `log_events`, so on the deploy that first adds the table the worker may start before the API
|
||||||
# has migrated — those few seconds of batches fail and say so on stderr; nothing else breaks.
|
# has migrated — those few seconds of batches fail and say so on stderr; nothing else breaks.
|
||||||
logsink.start()
|
logsink.start()
|
||||||
|
problem = storage.files_root_problem(settings.files_root)
|
||||||
|
if problem:
|
||||||
|
# Refuse, loudly: downloading into a root that is not the volume loses every file with the
|
||||||
|
# container. The queue waits until the mount is fixed (see docs/self-hosting.md).
|
||||||
|
log.critical("The files library cannot be written: %s. The worker will not start.", problem)
|
||||||
|
raise SystemExit(1)
|
||||||
_recover_orphans()
|
_recover_orphans()
|
||||||
global _target
|
global _target
|
||||||
_target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer
|
_target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer
|
||||||
threads: dict[int, threading.Thread] = {}
|
threads: dict[int, threading.Thread] = {}
|
||||||
level = _sync_knobs(threads, None)
|
level = _sync_knobs(threads, None)
|
||||||
log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.download_root)
|
log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.files_root)
|
||||||
while not _stop.wait(_KNOB_SECONDS):
|
while not _stop.wait(_KNOB_SECONDS):
|
||||||
level = _sync_knobs(threads, level)
|
level = _sync_knobs(threads, level)
|
||||||
log.info("Download worker stopping…")
|
log.info("Download worker stopping…")
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ services:
|
|||||||
POSTGRES_DB: siftlode_test
|
POSTGRES_DB: siftlode_test
|
||||||
tmpfs:
|
tmpfs:
|
||||||
- /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run
|
- /var/lib/postgresql/data # ephemeral + fast; the schema is rebuilt per run
|
||||||
|
# One database per pytest-xdist worker, each through the app's own pool (5 + 10 overflow): the default 100
|
||||||
|
# connections would run out at 8 workers.
|
||||||
|
command: ["postgres", "-c", "max_connections=300"]
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD-SHELL", "pg_isready -U siftlode -d siftlode_test"]
|
test: ["CMD-SHELL", "pg_isready -U siftlode -d siftlode_test"]
|
||||||
interval: 1s
|
interval: 1s
|
||||||
@@ -37,4 +40,4 @@ services:
|
|||||||
- ./:/app
|
- ./:/app
|
||||||
working_dir: /app
|
working_dir: /app
|
||||||
# Coverage is on demand, not in the gate: see docs/development.md.
|
# Coverage is on demand, not in the gate: see docs/development.md.
|
||||||
command: ["python", "-m", "pytest"]
|
command: ["python", "-m", "pytest", "-n", "auto"]
|
||||||
|
|||||||
+6
-1
@@ -7,4 +7,9 @@
|
|||||||
# not an unknown marker that quietly does nothing.
|
# not an unknown marker that quietly does nothing.
|
||||||
testpaths = tests
|
testpaths = tests
|
||||||
pythonpath = .
|
pythonpath = .
|
||||||
addopts = -q -rs --strict-markers
|
# `-p tests.db_per_worker`: under pytest-xdist (`-n`) every worker gets its own database — DATABASE_URL + `_gw<N>`,
|
||||||
|
# created on first use — because the `db` fixture's TRUNCATE-per-test isolation assumes one process per database.
|
||||||
|
# It is a plugin so it runs before conftest imports the app (whose settings read DATABASE_URL once). Serial: no-op.
|
||||||
|
# The file is a verbatim copy of a shared tool (see its docstring); change the source, not the copy.
|
||||||
|
addopts = -q -rs --strict-markers -p tests.db_per_worker
|
||||||
|
db_per_worker_env = DATABASE_URL
|
||||||
|
|||||||
@@ -16,3 +16,6 @@ pytest-cov==7.1.0
|
|||||||
ruff==0.15.21
|
ruff==0.15.21
|
||||||
httpx2==2.13.0
|
httpx2==2.13.0
|
||||||
pip-audit==2.10.1
|
pip-audit==2.10.1
|
||||||
|
# pytest-xdist: the DB lane runs on every core (`-n auto` in docker-compose.test.yml); tests/db_per_worker.py gives
|
||||||
|
# each worker its own database. Same tests, same strictness — only side by side.
|
||||||
|
pytest-xdist==3.8.0
|
||||||
|
|||||||
@@ -2,9 +2,10 @@
|
|||||||
# Source of truth is requirements.txt (the floor ranges); regenerate after editing it with:
|
# Source of truth is requirements.txt (the floor ranges); regenerate after editing it with:
|
||||||
# docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort
|
# docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort
|
||||||
# The prod Dockerfile + Dockerfile.test install THIS file; the gate's pip-audit (scripts/check.sh)
|
# The prod Dockerfile + Dockerfile.test install THIS file; the gate's pip-audit (scripts/check.sh)
|
||||||
# scans it, blocking. A security fix may be ONE pin bumped in place instead of a full
|
# scans it, blocking. A security fix, or a yt-dlp release YouTube's changes force, may be ONE pin
|
||||||
# regeneration, which would move every other package too — then `pip install -r requirements.lock && pip check` must
|
# bumped in place instead of a full regeneration, which would move every other package too — then
|
||||||
# pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177).
|
# `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19
|
||||||
|
# (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403).
|
||||||
APScheduler==3.11.3
|
APScheduler==3.11.3
|
||||||
Authlib==1.7.2
|
Authlib==1.7.2
|
||||||
Mako==1.4.1
|
Mako==1.4.1
|
||||||
@@ -62,4 +63,4 @@ uvloop==0.22.1
|
|||||||
watchfiles==1.2.0
|
watchfiles==1.2.0
|
||||||
websockets==17.0.1
|
websockets==17.0.1
|
||||||
yt-dlp-ejs==0.8.0
|
yt-dlp-ejs==0.8.0
|
||||||
yt-dlp==2026.7.4
|
yt-dlp==2026.8.19
|
||||||
|
|||||||
@@ -16,12 +16,14 @@ argon2-cffi>=23.1,<26
|
|||||||
# Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile)
|
# Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile)
|
||||||
# does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver
|
# does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver
|
||||||
# scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so
|
# scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so
|
||||||
# keep the floor loose and bump it when extraction breaks.
|
# keep the floor loose and bump it when extraction breaks. 2026.8.19 marks the clients that need a
|
||||||
yt-dlp[default]>=2025.5.22
|
# stream PO token, which the shipped player clients (config.download_player_clients) rely on.
|
||||||
|
yt-dlp[default]>=2026.8.19
|
||||||
# Browser impersonation (`impersonate`) for Facebook downloads made with the user's cookie: some
|
# Browser impersonation (`impersonate`) for Facebook downloads made with the user's cookie: some
|
||||||
# links answer a signed-in plain request with "Cannot parse data" and only parse as Chrome
|
# links answer a signed-in plain request with "Cannot parse data" and only parse as Chrome
|
||||||
# (measured). The range is YT-DLP's, not ours — yt-dlp 2026.07.04 imports curl_cffi only in
|
# (measured). The range is YT-DLP's, not ours — yt-dlp 2026.07.04 imported curl_cffi only in
|
||||||
# 0.10.x–0.15.x and silently reports every target "unavailable" outside it. Move it when bumping
|
# 0.10.x–0.15.x and silently reported every target "unavailable" outside it (2026.08.19 takes up to
|
||||||
|
# 0.16.x; the cap stays on the measured 0.15 until a bump re-measures it). Move it when bumping
|
||||||
# yt-dlp; `tests/test_facebook_impersonation.py::test_the_locked_curl_cffi_can_impersonate_chrome`
|
# yt-dlp; `tests/test_facebook_impersonation.py::test_the_locked_curl_cffi_can_impersonate_chrome`
|
||||||
# fails the gate if the locked pair can no longer impersonate.
|
# fails the gate if the locked pair can no longer impersonate.
|
||||||
curl_cffi>=0.10,<0.16
|
curl_cffi>=0.10,<0.16
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ from starlette.requests import HTTPConnection
|
|||||||
|
|
||||||
import app.models # register every model on Base.metadata
|
import app.models # register every model on Base.metadata
|
||||||
import app.state as app_state
|
import app.state as app_state
|
||||||
from app import security
|
from app import apitokens, security
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import Base, SessionLocal, engine, get_db
|
from app.db import Base, SessionLocal, engine, get_db
|
||||||
@@ -242,8 +242,16 @@ def as_user(db, override_dep, _configured):
|
|||||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||||
return registry[uid]
|
return registry[uid]
|
||||||
|
|
||||||
|
def _session_or_token(request: Request) -> User | None:
|
||||||
|
# The Files routes' session half (`apitokens.session_caller`): a request carrying a token is
|
||||||
|
# left to the real token check, exactly as in production.
|
||||||
|
if apitokens.bearer_token(request) is not None:
|
||||||
|
return None
|
||||||
|
return _resolver(request)
|
||||||
|
|
||||||
override_dep(get_db, db.info["request"])
|
override_dep(get_db, db.info["request"])
|
||||||
override_dep(current_user, _resolver)
|
override_dep(current_user, _resolver)
|
||||||
|
override_dep(apitokens.session_caller, _session_or_token)
|
||||||
|
|
||||||
def _as(user: User) -> TestClient:
|
def _as(user: User) -> TestClient:
|
||||||
registry[user.id] = user
|
registry[user.id] = user
|
||||||
@@ -254,20 +262,24 @@ def as_user(db, override_dep, _configured):
|
|||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def upload_headers():
|
def upload_headers():
|
||||||
"""Build the headers `POST /api/downloads/upload` takes (the file is the raw body).
|
"""Build the headers the upload routes take (the file is the raw body).
|
||||||
|
|
||||||
Shared, because two test modules drive that route and a second copy of this is exactly the drift
|
Shared, because two test modules drive that route and a second copy of this is exactly the drift
|
||||||
that lets the wire shape change in one place only. The body IS the file now; the names ride as
|
that lets the wire shape change in one place only. The body IS the file now; the names ride as
|
||||||
percent-encoded headers (`quote` mirrors the client's `encodeURIComponent`, since headers are
|
percent-encoded headers (`quote` mirrors the client's `encodeURIComponent`, since headers are
|
||||||
latin-1 and a Hungarian filename would not survive raw)."""
|
latin-1 and a Hungarian filename would not survive raw)."""
|
||||||
|
|
||||||
def _headers(filename: str, display_name: str | None = None) -> dict[str, str]:
|
def _headers(
|
||||||
|
filename: str, display_name: str | None = None, folder: int | str | None = None
|
||||||
|
) -> dict[str, str]:
|
||||||
headers = {
|
headers = {
|
||||||
"content-type": "application/octet-stream",
|
"content-type": "application/octet-stream",
|
||||||
"x-upload-filename": quote(filename),
|
"x-upload-filename": quote(filename),
|
||||||
}
|
}
|
||||||
if display_name is not None:
|
if display_name is not None:
|
||||||
headers["x-upload-display-name"] = quote(display_name)
|
headers["x-upload-display-name"] = quote(display_name)
|
||||||
|
if folder is not None:
|
||||||
|
headers["x-upload-folder"] = str(folder)
|
||||||
return headers
|
return headers
|
||||||
|
|
||||||
return _headers
|
return _headers
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
"""Give every pytest-xdist worker its own database, so a suite that shares one test database can run in parallel.
|
||||||
|
|
||||||
|
Generic (source of truth: the memory repo's tools/pytest-parallel/db_per_worker.py); a project carries a verbatim copy
|
||||||
|
next to its conftest — tests often run in a container that cannot see this repo — and
|
||||||
|
`tools/pytest-parallel/sync-check.sh <copy>` reports drift. Keep the copy byte-identical; change it here.
|
||||||
|
|
||||||
|
Use, as a pytest plugin (it then runs before any conftest, i.e. before the app reads its settings):
|
||||||
|
pytest.ini: addopts = -p tests.db_per_worker (the copy's import path; `python -m pytest` puts cwd on sys.path)
|
||||||
|
db_per_worker_env = DATABASE_URL (the environment variable that holds the test database)
|
||||||
|
or call `apply("<ENV VAR>")` yourself before the app is imported. The variable may hold:
|
||||||
|
- a Postgres URL (`postgresql://…/name`, `postgresql+psycopg://…`): the worker gets `name_gw<N>`, created from the
|
||||||
|
base database's server if missing (empty — the project's own schema fixture migrates it, as for one process);
|
||||||
|
- a SQLite URL (`sqlite:///path.db`) or a plain file path: the worker gets `path-gw<N>.db`.
|
||||||
|
Without xdist (no PYTEST_XDIST_WORKER) or with the variable unset it changes nothing, so a serial run is unaffected.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from urllib.parse import urlsplit, urlunsplit
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
_PG = re.compile(r"^postgres(?:ql)?(?:\+\w+)?://", re.I)
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_addoption(parser: pytest.Parser) -> None:
|
||||||
|
parser.addini(
|
||||||
|
"db_per_worker_env",
|
||||||
|
"env var holding the test database (a Postgres URL, SQLite URL or file path)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.hookimpl(tryfirst=True)
|
||||||
|
def pytest_load_initial_conftests(early_config: pytest.Config) -> None:
|
||||||
|
var = early_config.getini("db_per_worker_env")
|
||||||
|
if var:
|
||||||
|
apply(str(var))
|
||||||
|
|
||||||
|
|
||||||
|
def apply(var: str) -> str | None:
|
||||||
|
worker = os.environ.get("PYTEST_XDIST_WORKER")
|
||||||
|
value = os.environ.get(var)
|
||||||
|
if not worker or not value:
|
||||||
|
return value
|
||||||
|
if _PG.match(value):
|
||||||
|
new = _postgres(value, worker)
|
||||||
|
else:
|
||||||
|
new = _file(value, worker)
|
||||||
|
os.environ[var] = new
|
||||||
|
return new
|
||||||
|
|
||||||
|
|
||||||
|
def _postgres(url: str, worker: str) -> str:
|
||||||
|
parts = urlsplit(url)
|
||||||
|
base = parts.path.lstrip("/")
|
||||||
|
name = f"{base}_{worker}"
|
||||||
|
if not re.fullmatch(r"\w+", name):
|
||||||
|
raise ValueError(f"db_per_worker: refusing an unusual database name {name!r}")
|
||||||
|
# The server, through the BASE database (it exists; `postgres` may not be reachable for this role).
|
||||||
|
plain = urlunsplit(
|
||||||
|
(re.sub(r"\+\w+$", "", parts.scheme), parts.netloc, parts.path, parts.query, "")
|
||||||
|
)
|
||||||
|
_create_if_missing(plain, name)
|
||||||
|
return urlunsplit((parts.scheme, parts.netloc, f"/{name}", parts.query, parts.fragment))
|
||||||
|
|
||||||
|
|
||||||
|
def _create_if_missing(dsn: str, name: str) -> None:
|
||||||
|
try:
|
||||||
|
import psycopg # psycopg 3
|
||||||
|
|
||||||
|
conn = psycopg.connect(dsn, autocommit=True)
|
||||||
|
except ImportError:
|
||||||
|
import psycopg2 # fallback for projects on psycopg2
|
||||||
|
|
||||||
|
conn = psycopg2.connect(dsn)
|
||||||
|
conn.autocommit = True
|
||||||
|
try:
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (name,))
|
||||||
|
if cur.fetchone() is None:
|
||||||
|
cur.execute(f'CREATE DATABASE "{name}"') # name checked to be \w+ above
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _file(value: str, worker: str) -> str:
|
||||||
|
prefix = ""
|
||||||
|
m = re.match(r"^(sqlite(?:\+\w+)?:///)(.*)$", value, re.I)
|
||||||
|
if m:
|
||||||
|
prefix, value = m.group(1), m.group(2)
|
||||||
|
root, ext = os.path.splitext(value)
|
||||||
|
return f"{prefix}{root}-{worker}{ext}"
|
||||||
@@ -175,6 +175,21 @@ def test_the_cookie_capable_clients_match_the_installed_ytdlp():
|
|||||||
assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live
|
assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_shipped_player_clients_exist_in_the_installed_ytdlp():
|
||||||
|
# yt-dlp skips a client it does not know with a warning and falls back to its own defaults, so
|
||||||
|
# a bump that renames or drops one (tv_simply, the one measured to deliver HD) would change
|
||||||
|
# what every download is fetched with, silently. Read from the Settings defaults, not from
|
||||||
|
# `settings`, which an environment may override.
|
||||||
|
from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS
|
||||||
|
|
||||||
|
from app.config import Settings
|
||||||
|
|
||||||
|
fields = Settings.model_fields
|
||||||
|
for knob in ("download_player_clients", "download_player_clients_fallback"):
|
||||||
|
names = Settings._clients(fields[knob].default)
|
||||||
|
assert names and set(names) <= set(INNERTUBE_CLIENTS), (knob, names)
|
||||||
|
|
||||||
|
|
||||||
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
|
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
|
||||||
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
|
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
|
||||||
assert ea["youtube"]["player_client"] == ["android_vr"]
|
assert ea["youtube"]["player_client"] == ["android_vr"]
|
||||||
|
|||||||
@@ -0,0 +1,648 @@
|
|||||||
|
"""API tokens for the Files library (`app.apitokens`, `routes.tokens`), DB-lane.
|
||||||
|
|
||||||
|
A token is made, listed and revoked by its signed-in user only; its value is in the answer that made
|
||||||
|
it and nowhere after. With it a machine reaches `/api/files` and nothing else, and there only the
|
||||||
|
folders its grants name, with the rights they give — every right checked in a folder it has and in
|
||||||
|
one it has not. A token that is malformed, unknown, expired, revoked, or whose account is suspended
|
||||||
|
holds nothing, and a client trying tokens is slowed down.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import Depends, Request
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from sqlalchemy import select, update
|
||||||
|
|
||||||
|
from app import apitokens, audit
|
||||||
|
from app.apitokens import files_caller
|
||||||
|
from app.auth import current_user
|
||||||
|
from app.downloads import folders, service
|
||||||
|
from app.downloads import gc as gcmod
|
||||||
|
from app.downloads import upload as uploadmod
|
||||||
|
from app.downloads.status import JobStatus
|
||||||
|
from app.config import settings
|
||||||
|
from app.db import SessionLocal, get_db
|
||||||
|
from app.main import app
|
||||||
|
from app.notifications import create_notification
|
||||||
|
from app.models import (
|
||||||
|
ApiToken,
|
||||||
|
ApiTokenGrant,
|
||||||
|
AuditLog,
|
||||||
|
DownloadJob,
|
||||||
|
FileFolder,
|
||||||
|
MediaAsset,
|
||||||
|
Notification,
|
||||||
|
User,
|
||||||
|
)
|
||||||
|
from app.ratelimit import RateLimiter
|
||||||
|
from app.routes import tokens as tokens_routes
|
||||||
|
from app.routes.auth import bump_session_epoch
|
||||||
|
from app.security import hash_token
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def _root(tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def _no_probe(monkeypatch):
|
||||||
|
# A file ffprobe cannot read: a generic file, done at once (no conversion).
|
||||||
|
monkeypatch.setattr(uploadmod.probe, "probe_json", lambda _p: None)
|
||||||
|
|
||||||
|
|
||||||
|
def _own(db, user_id, name):
|
||||||
|
folder = FileFolder(user_id=user_id, kind=folders.FolderKind.OWN, name=name)
|
||||||
|
db.add(folder)
|
||||||
|
db.commit()
|
||||||
|
return folder
|
||||||
|
|
||||||
|
|
||||||
|
def _file_in(db, root, user_id, folder, name):
|
||||||
|
"""A done file of `user_id`'s with bytes on disk, sitting in `folder`."""
|
||||||
|
asset = MediaAsset(
|
||||||
|
source_kind="upload",
|
||||||
|
source_ref=name,
|
||||||
|
format_sig=f"sig-{name}",
|
||||||
|
status="ready",
|
||||||
|
rel_path=f"{name}.bin",
|
||||||
|
size_bytes=4,
|
||||||
|
ref_count=0,
|
||||||
|
title=name,
|
||||||
|
media_kind="file",
|
||||||
|
container="bin",
|
||||||
|
)
|
||||||
|
db.add(asset)
|
||||||
|
db.flush()
|
||||||
|
job = DownloadJob(
|
||||||
|
user_id=user_id,
|
||||||
|
asset_id=asset.id,
|
||||||
|
source_kind="upload",
|
||||||
|
source_ref=name,
|
||||||
|
profile_snapshot={},
|
||||||
|
status=JobStatus.DONE,
|
||||||
|
)
|
||||||
|
db.add(job)
|
||||||
|
db.flush()
|
||||||
|
folders.place(db, job)
|
||||||
|
job.folder_id = folder.id
|
||||||
|
folders.stamp_done(db, job)
|
||||||
|
service.incr_ref_count(db, asset.id)
|
||||||
|
db.commit()
|
||||||
|
(root / f"{name}.bin").write_bytes(b"data")
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def _bearer(raw: str) -> TestClient:
|
||||||
|
return TestClient(app, headers={"authorization": f"Bearer {raw}"})
|
||||||
|
|
||||||
|
|
||||||
|
def _make(c, **body) -> dict:
|
||||||
|
r = c.post("/api/tokens", json={"name": "a machine", **body})
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
return r.json()
|
||||||
|
|
||||||
|
|
||||||
|
def _audits(db, action):
|
||||||
|
return db.execute(select(AuditLog).where(AuditLog.action == action)).scalars().all()
|
||||||
|
|
||||||
|
|
||||||
|
# --- the token itself ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_has_its_shape_and_a_checksum_that_refuses_a_typo():
|
||||||
|
raw = apitokens.mint()
|
||||||
|
assert raw.startswith("sfl_") and len(raw) == apitokens.TOKEN_LEN
|
||||||
|
assert apitokens.well_formed(raw)
|
||||||
|
assert apitokens.mint() != raw
|
||||||
|
flipped = raw[:10] + ("A" if raw[10] != "A" else "B") + raw[11:]
|
||||||
|
assert not apitokens.well_formed(flipped)
|
||||||
|
assert not apitokens.well_formed(raw[:-1])
|
||||||
|
assert not apitokens.well_formed("xxx_" + raw[4:])
|
||||||
|
# A header byte past ASCII arrives as one latin-1 character: refused, never a 500.
|
||||||
|
assert not apitokens.well_formed(raw[:-1] + "é")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_non_ascii_token_is_refused_as_any_bad_one(db, client):
|
||||||
|
# The test client re-sends a high byte as two UTF-8 bytes, which arrive as two latin-1
|
||||||
|
# characters: one "A" fewer keeps the header a token's length, so it is the checksum that meets
|
||||||
|
# them, not the length check.
|
||||||
|
raw = ("Bearer sfl_" + "A" * 47 + "é").encode("latin-1")
|
||||||
|
r = TestClient(app, headers={"authorization": raw}).get("/api/files/folders")
|
||||||
|
assert r.status_code == 401 and r.json()["detail"]["reason"] == "not_authenticated"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_hub_preset_makes_one_hub_folder_with_every_right_and_shows_the_token_once(
|
||||||
|
db, make_user, as_user
|
||||||
|
):
|
||||||
|
alice = make_user()
|
||||||
|
c = as_user(alice)
|
||||||
|
made = _make(c, preset="hub")
|
||||||
|
raw = made["token"]
|
||||||
|
assert apitokens.well_formed(raw) and made["prefix"] == raw[: apitokens.SHOWN_LEN]
|
||||||
|
[grant] = made["grants"]
|
||||||
|
assert grant["kind"] == "hub" and grant["rights"] == ["delete", "list", "read", "write"]
|
||||||
|
# Only the hash is kept, and the value is not in the list.
|
||||||
|
stored = db.execute(select(ApiToken)).scalar_one()
|
||||||
|
assert stored.token_hash != raw and raw not in str(c.get("/api/tokens").json())
|
||||||
|
# A second hub token reuses the one Hub folder.
|
||||||
|
again = _make(c, preset="hub")
|
||||||
|
assert again["grants"][0]["folder_id"] == grant["folder_id"]
|
||||||
|
hubs = db.execute(select(FileFolder).where(FileFolder.kind == "hub")).scalars().all()
|
||||||
|
assert len(hubs) == 1
|
||||||
|
assert len(_audits(db, "api_token_create")) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_is_granted_only_the_users_own_folders_and_writes_only_where_uploads_land(
|
||||||
|
db, make_user, as_user
|
||||||
|
):
|
||||||
|
alice, bob = make_user(), make_user()
|
||||||
|
theirs = _own(db, bob.id, "Bob's")
|
||||||
|
c = as_user(alice)
|
||||||
|
r = c.post(
|
||||||
|
"/api/tokens",
|
||||||
|
json={"name": "x", "grants": [{"folder_id": theirs.id, "rights": ["read"]}]},
|
||||||
|
)
|
||||||
|
assert r.status_code == 409 and r.json()["detail"]["reason"] == "file_folder_gone"
|
||||||
|
youtube = folders.source_folder(db, alice.id, folders.SourceKey.YOUTUBE)
|
||||||
|
db.commit()
|
||||||
|
r = c.post(
|
||||||
|
"/api/tokens",
|
||||||
|
json={"name": "x", "grants": [{"folder_id": youtube.id, "rights": ["write"]}]},
|
||||||
|
)
|
||||||
|
assert r.json()["detail"]["reason"] == "file_folder_not_its_source"
|
||||||
|
assert c.post("/api/tokens", json={"name": "x"}).json()["detail"]["reason"] == (
|
||||||
|
"api_token_no_grants"
|
||||||
|
)
|
||||||
|
assert db.execute(select(ApiToken)).all() == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("right", ["read", "write"])
|
||||||
|
def test_a_folder_granted_is_held_until_the_token_is_in(db, make_user, right):
|
||||||
|
"""A delete of the folder at the same moment waits and takes the new grant with it, rather than
|
||||||
|
failing the grant's foreign key."""
|
||||||
|
import threading
|
||||||
|
|
||||||
|
alice = make_user()
|
||||||
|
fid = _own(db, alice.id, "Mine").id
|
||||||
|
body = tokens_routes.TokenIn(
|
||||||
|
name="x", grants=[tokens_routes.GrantIn(folder_id=fid, rights=[right])]
|
||||||
|
)
|
||||||
|
tokens_routes._wanted_grants(db, alice, body)
|
||||||
|
outcome: list[object] = []
|
||||||
|
|
||||||
|
def delete_it():
|
||||||
|
other = SessionLocal()
|
||||||
|
try:
|
||||||
|
outcome.append(folders.delete_own(other, other.get(FileFolder, fid)))
|
||||||
|
other.commit()
|
||||||
|
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
|
||||||
|
outcome.append(e)
|
||||||
|
finally:
|
||||||
|
other.close()
|
||||||
|
|
||||||
|
t = threading.Thread(target=delete_it)
|
||||||
|
t.start()
|
||||||
|
t.join(timeout=1)
|
||||||
|
assert t.is_alive() # the delete waits for the token on its way in
|
||||||
|
db.commit()
|
||||||
|
t.join(timeout=10)
|
||||||
|
assert not isinstance(outcome[0], Exception), outcome
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_user_has_at_most_the_ceiling_of_tokens(db, make_user, as_user, monkeypatch):
|
||||||
|
monkeypatch.setattr(tokens_routes, "MAX_TOKENS", 2)
|
||||||
|
c = as_user(make_user())
|
||||||
|
_make(c, preset="hub")
|
||||||
|
_make(c, preset="hub")
|
||||||
|
r = c.post("/api/tokens", json={"name": "third", "preset": "hub"})
|
||||||
|
assert r.status_code == 409 and r.json()["detail"]["reason"] == "api_token_limit"
|
||||||
|
|
||||||
|
|
||||||
|
# --- what a token reaches ------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("right", ["list", "read", "write", "delete"])
|
||||||
|
def test_each_right_holds_in_its_folder_only(
|
||||||
|
db, make_user, as_user, _root, _no_probe, upload_headers, right
|
||||||
|
):
|
||||||
|
"""One right in folder A; folder B has none. Every Files route a token may use, in both."""
|
||||||
|
alice = make_user()
|
||||||
|
a, b = _own(db, alice.id, "A"), _own(db, alice.id, "B")
|
||||||
|
in_a, in_b = _file_in(db, _root, alice.id, a, "fa"), _file_in(db, _root, alice.id, b, "fb")
|
||||||
|
raw = _make(as_user(alice), grants=[{"folder_id": a.id, "rights": [right]}])["token"]
|
||||||
|
m = _bearer(raw)
|
||||||
|
|
||||||
|
folders_seen = [f["id"] for f in m.get("/api/files/folders").json()]
|
||||||
|
assert folders_seen == ([a.id] if right == "list" else [])
|
||||||
|
listed = m.get("/api/files", params={"folder_id": a.id})
|
||||||
|
assert listed.status_code == (200 if right == "list" else 404)
|
||||||
|
if right == "list":
|
||||||
|
assert [f["id"] for f in listed.json()] == [in_a.id]
|
||||||
|
assert [f["id"] for f in m.get("/api/files").json()] == [in_a.id]
|
||||||
|
else:
|
||||||
|
assert m.get("/api/files").json() == []
|
||||||
|
assert m.get("/api/files", params={"folder_id": b.id}).status_code == 404
|
||||||
|
|
||||||
|
read_a = m.get(f"/api/files/{in_a.id}/file")
|
||||||
|
assert read_a.status_code == (200 if right == "read" else 404)
|
||||||
|
assert m.get(f"/api/files/{in_a.id}/sha256").status_code == (200 if right == "read" else 404)
|
||||||
|
assert m.get(f"/api/files/{in_b.id}/file").status_code == 404
|
||||||
|
|
||||||
|
up_a = m.post("/api/files/upload", content=b"hi", headers=upload_headers("n.txt", folder=a.id))
|
||||||
|
assert up_a.status_code == (200 if right == "write" else 404)
|
||||||
|
if right == "write":
|
||||||
|
assert up_a.json()["folder_id"] == a.id
|
||||||
|
up_b = m.post("/api/files/upload", content=b"hi", headers=upload_headers("n.txt", folder=b.id))
|
||||||
|
assert up_b.status_code == 404
|
||||||
|
no_folder = m.post("/api/files/upload", content=b"hi", headers=upload_headers("n.txt"))
|
||||||
|
assert no_folder.status_code == 400
|
||||||
|
assert no_folder.json()["detail"]["reason"] == "api_token_folder_required"
|
||||||
|
|
||||||
|
assert m.delete(f"/api/files/{in_b.id}").status_code == 404
|
||||||
|
assert db.get(DownloadJob, in_b.id) is not None
|
||||||
|
del_a = m.delete(f"/api/files/{in_a.id}")
|
||||||
|
assert del_a.status_code == (200 if right == "delete" else 404)
|
||||||
|
db.expire_all()
|
||||||
|
assert (db.get(DownloadJob, in_a.id) is None) == (right == "delete")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_never_reaches_another_users_files_or_the_shared_ones(
|
||||||
|
db, make_user, as_user, _root
|
||||||
|
):
|
||||||
|
alice, bob = make_user(), make_user()
|
||||||
|
hub_raw = _make(as_user(alice), preset="hub")["token"]
|
||||||
|
bobs = _file_in(db, _root, bob.id, _own(db, bob.id, "B"), "bf")
|
||||||
|
m = _bearer(hub_raw)
|
||||||
|
assert m.get(f"/api/files/{bobs.id}/file").status_code == 404
|
||||||
|
assert m.delete(f"/api/files/{bobs.id}").status_code == 404
|
||||||
|
shared = m.get("/api/files", params={"scope": "shared"})
|
||||||
|
assert shared.status_code == 403 and shared.json()["detail"]["reason"] == "api_token_scope"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_is_judged_by_its_grants_even_beside_a_signed_in_cookie(db, make_user, client):
|
||||||
|
"""Through the real session stack (`client`, a real sign-in): the `as_user` fixture stands in
|
||||||
|
for the very dependency this is about."""
|
||||||
|
alice = make_user(email="alice@example.com")
|
||||||
|
a = _own(db, alice.id, "A")
|
||||||
|
c = client
|
||||||
|
signed_in = c.post(
|
||||||
|
"/auth/password-login",
|
||||||
|
json={"email": "alice@example.com", "password": "correct horse battery"},
|
||||||
|
)
|
||||||
|
assert signed_in.status_code == 200
|
||||||
|
raw = _make(c, grants=[{"folder_id": a.id, "rights": ["read"]}])["token"]
|
||||||
|
assert len(c.get("/api/files/folders").json()) == 1 # the user sees every folder
|
||||||
|
both = c.get("/api/files/folders", headers={"authorization": f"Bearer {raw}"})
|
||||||
|
assert both.json() == [] # the token may not list: its grants decide
|
||||||
|
|
||||||
|
|
||||||
|
def _signed_in(client, email="alice@example.com") -> TestClient:
|
||||||
|
r = client.post(
|
||||||
|
"/auth/password-login", json={"email": email, "password": "correct horse battery"}
|
||||||
|
)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_works_on_the_files_routes_only(db, make_user, client):
|
||||||
|
"""Through the real session stack: under `as_user` every route without its header answers 401
|
||||||
|
whatever `current_user` does with a token, so these refusals would prove nothing there."""
|
||||||
|
make_user(email="alice@example.com")
|
||||||
|
raw = _make(_signed_in(client), preset="hub")["token"]
|
||||||
|
m = _bearer(raw)
|
||||||
|
assert m.get("/api/files/folders").status_code == 200
|
||||||
|
# Making a wider token, or reading the account, takes the user's session.
|
||||||
|
assert m.get("/api/me").json() == {"authenticated": False} # the token is no one here
|
||||||
|
assert m.get("/api/tokens").status_code == 401
|
||||||
|
assert m.post("/api/tokens", json={"name": "wider", "preset": "hub"}).status_code == 401
|
||||||
|
assert m.get("/api/files/usage").status_code == 401
|
||||||
|
assert m.patch("/api/files/1", json={"name": "x"}).status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_the_files_routes_take_a_token():
|
||||||
|
"""The route table, not a list kept by hand: every route that resolves `files_caller` is under
|
||||||
|
/api/files, and these are all of them. The walk recurses into included routers (see
|
||||||
|
`test_body_limit._declared_routes` for why `app.routes` alone sees none of ours)."""
|
||||||
|
takers, nodes, routes = set(), [app], []
|
||||||
|
while nodes:
|
||||||
|
for route in getattr(nodes.pop(), "routes", ()) or ():
|
||||||
|
inner = getattr(route, "original_router", None)
|
||||||
|
if inner is not None:
|
||||||
|
nodes.append(inner)
|
||||||
|
else:
|
||||||
|
routes.append(route)
|
||||||
|
assert len(routes) > 100 # the walk found the app's routes, not just the shell's
|
||||||
|
for route in routes:
|
||||||
|
dependant = getattr(route, "dependant", None)
|
||||||
|
if dependant is None:
|
||||||
|
continue
|
||||||
|
stack, calls = [dependant], set()
|
||||||
|
while stack:
|
||||||
|
d = stack.pop()
|
||||||
|
calls.add(d.call)
|
||||||
|
stack.extend(d.dependencies)
|
||||||
|
if files_caller in calls:
|
||||||
|
for method in route.methods:
|
||||||
|
takers.add((method, route.path))
|
||||||
|
assert all(path.startswith("/api/files") for _, path in takers)
|
||||||
|
assert takers == {
|
||||||
|
("GET", "/api/files/folders"),
|
||||||
|
("GET", "/api/files"),
|
||||||
|
("GET", "/api/files/{job_id}/file"),
|
||||||
|
("GET", "/api/files/{job_id}/sha256"),
|
||||||
|
("POST", "/api/files/upload"),
|
||||||
|
("DELETE", "/api/files/{job_id}"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_password(db, client, alice):
|
||||||
|
from app.routes.auth import _issue_token
|
||||||
|
|
||||||
|
link = _issue_token(db, alice, "reset", timedelta(hours=1))
|
||||||
|
r = client.post(
|
||||||
|
"/auth/password-reset/confirm", json={"token": link, "password": "a brand new password"}
|
||||||
|
)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
|
||||||
|
|
||||||
|
def _change_password(db, client, alice):
|
||||||
|
r = client.post(
|
||||||
|
"/auth/set-password",
|
||||||
|
json={"password": "a brand new password", "current_password": "correct horse battery"},
|
||||||
|
)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
|
||||||
|
|
||||||
|
def _log_out_others(db, client, alice):
|
||||||
|
assert client.post("/auth/logout-others").status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def _revoke_untrusted_password(db, client, alice):
|
||||||
|
from app.routes.auth import _revoke_password
|
||||||
|
|
||||||
|
_revoke_password(db, alice)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"end_sessions",
|
||||||
|
[_reset_password, _change_password, _log_out_others, _revoke_untrusted_password],
|
||||||
|
)
|
||||||
|
def test_whatever_ends_every_session_ends_every_token_too(db, make_user, client, end_sessions):
|
||||||
|
"""A token minted from a stolen session must not outlive the owner's answer to the theft."""
|
||||||
|
alice = make_user(email="alice@example.com")
|
||||||
|
bob = make_user(email="bob@example.com")
|
||||||
|
raw = _make(_signed_in(client), preset="hub")["token"]
|
||||||
|
bobs = apitokens.mint()
|
||||||
|
db.add(ApiToken(user_id=bob.id, name="b", prefix=bobs[:12], token_hash=hash_token(bobs)))
|
||||||
|
db.commit()
|
||||||
|
assert _bearer(raw).get("/api/files/folders").status_code == 200
|
||||||
|
end_sessions(db, client, alice)
|
||||||
|
assert _bearer(raw).get("/api/files/folders").status_code == 401
|
||||||
|
assert db.execute(select(ApiToken.user_id)).scalars().all() == [bob.id] # only hers went
|
||||||
|
[revoked] = _audits(db, "api_token_revoke")
|
||||||
|
assert revoked.actor_id == alice.id and "every session" in (revoked.summary or "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_asked_for_as_every_session_ends_is_refused(db, make_user, client, override_dep):
|
||||||
|
"""The session was still good when its request came in, and ended before the token was made."""
|
||||||
|
alice = make_user(email="alice@example.com")
|
||||||
|
c = _signed_in(client)
|
||||||
|
|
||||||
|
def ended_meanwhile(request: Request, session=Depends(get_db)):
|
||||||
|
user = current_user(request, session)
|
||||||
|
other = SessionLocal()
|
||||||
|
try:
|
||||||
|
bump_session_epoch(other, other.get(User, alice.id))
|
||||||
|
other.commit()
|
||||||
|
finally:
|
||||||
|
other.close()
|
||||||
|
return user
|
||||||
|
|
||||||
|
override_dep(current_user, ended_meanwhile)
|
||||||
|
r = c.post("/api/tokens", json={"name": "late", "preset": "hub"})
|
||||||
|
assert r.status_code == 401
|
||||||
|
assert db.execute(select(ApiToken.id)).scalars().all() == []
|
||||||
|
|
||||||
|
|
||||||
|
def _meanwhile(work):
|
||||||
|
"""Run `work(other_session)` in another thread and session, committed; (thread, outcome)."""
|
||||||
|
import threading
|
||||||
|
|
||||||
|
outcome: list[object] = []
|
||||||
|
|
||||||
|
def run():
|
||||||
|
other = SessionLocal()
|
||||||
|
try:
|
||||||
|
outcome.append(work(other))
|
||||||
|
other.commit()
|
||||||
|
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
|
||||||
|
outcome.append(e)
|
||||||
|
finally:
|
||||||
|
other.close()
|
||||||
|
|
||||||
|
t = threading.Thread(target=run)
|
||||||
|
t.start()
|
||||||
|
return t, outcome
|
||||||
|
|
||||||
|
|
||||||
|
def test_ending_every_session_never_deadlocks_with_a_tokens_expiry_notice(db, make_user):
|
||||||
|
"""The notice's sweep holds the token's row and then writes a row naming the user; the sessions'
|
||||||
|
end holds the user and then deletes the token. A lock on the user that a mere reference to it
|
||||||
|
waits on would make that a deadlock — one of the two failing."""
|
||||||
|
alice = make_user()
|
||||||
|
raw = apitokens.mint()
|
||||||
|
soon = datetime.now(timezone.utc) + timedelta(days=1)
|
||||||
|
tok = ApiToken(
|
||||||
|
user_id=alice.id, name="t", prefix=raw[:12], token_hash=hash_token(raw), expires_at=soon
|
||||||
|
)
|
||||||
|
db.add(tok)
|
||||||
|
db.commit()
|
||||||
|
# `warn_expiring`, its two halves held apart: the token row taken…
|
||||||
|
db.execute(update(ApiToken).where(ApiToken.id == tok.id).values(expiry_warned=True))
|
||||||
|
t, outcome = _meanwhile(lambda other: bump_session_epoch(other, other.get(User, alice.id)))
|
||||||
|
t.join(timeout=1)
|
||||||
|
assert t.is_alive() # the sessions' end waits for the token row
|
||||||
|
# …and then its notice, naming the user.
|
||||||
|
create_notification(db, alice.id, "api_token_expiring", title="t", commit=False)
|
||||||
|
db.commit()
|
||||||
|
t.join(timeout=10)
|
||||||
|
assert outcome == [None]
|
||||||
|
db.expire_all()
|
||||||
|
assert db.execute(select(ApiToken.id)).scalars().all() == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_granted_a_folder_being_deleted_never_deadlocks(db, make_user):
|
||||||
|
"""The delete holds the folder and then writes its audit line naming the user; the token holds
|
||||||
|
the user and then waits for the folder."""
|
||||||
|
alice = make_user()
|
||||||
|
fid = _own(db, alice.id, "Mine").id
|
||||||
|
folders.delete_own(db, db.get(FileFolder, fid)) # the folder held, as the delete route does
|
||||||
|
body = tokens_routes.TokenIn(
|
||||||
|
name="x", grants=[tokens_routes.GrantIn(folder_id=fid, rights=["read"])]
|
||||||
|
)
|
||||||
|
t, outcome = _meanwhile(
|
||||||
|
lambda other: tokens_routes.create_token(body, other.get(User, alice.id), other)
|
||||||
|
)
|
||||||
|
t.join(timeout=1)
|
||||||
|
assert t.is_alive() # the token waits for the folder
|
||||||
|
audit.record(db, alice.id, "file_folder_delete", target_type="file_folder", target_id=fid)
|
||||||
|
db.commit()
|
||||||
|
t.join(timeout=10)
|
||||||
|
[refusal] = outcome
|
||||||
|
assert getattr(refusal, "status_code", None) == 409, refusal # the folder went first
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_made_as_every_session_ends_goes_with_them(db, make_user):
|
||||||
|
"""The other order: the token is on its way in when the sessions end."""
|
||||||
|
import threading
|
||||||
|
|
||||||
|
alice = make_user(email="alice@example.com")
|
||||||
|
db.execute(select(User.id).where(User.id == alice.id).with_for_update()) # as create_token
|
||||||
|
raw = apitokens.mint()
|
||||||
|
db.add(ApiToken(user_id=alice.id, name="t", prefix=raw[:12], token_hash=hash_token(raw)))
|
||||||
|
db.flush()
|
||||||
|
outcome: list[object] = []
|
||||||
|
|
||||||
|
def end_sessions():
|
||||||
|
other = SessionLocal()
|
||||||
|
try:
|
||||||
|
bump_session_epoch(other, other.get(User, alice.id))
|
||||||
|
other.commit()
|
||||||
|
outcome.append("done")
|
||||||
|
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
|
||||||
|
outcome.append(e)
|
||||||
|
finally:
|
||||||
|
other.close()
|
||||||
|
|
||||||
|
t = threading.Thread(target=end_sessions)
|
||||||
|
t.start()
|
||||||
|
t.join(timeout=1)
|
||||||
|
assert t.is_alive() # it waits for the token being made
|
||||||
|
db.commit()
|
||||||
|
t.join(timeout=10)
|
||||||
|
assert outcome == ["done"]
|
||||||
|
db.expire_all()
|
||||||
|
assert db.execute(select(ApiToken.id)).scalars().all() == []
|
||||||
|
|
||||||
|
|
||||||
|
# --- a token that does not hold ------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _refused(m: TestClient) -> int:
|
||||||
|
return m.get("/api/files/folders").status_code
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_malformed_unknown_expired_revoked_or_suspended_token_holds_nothing(
|
||||||
|
db, make_user, as_user
|
||||||
|
):
|
||||||
|
alice = make_user()
|
||||||
|
c = as_user(alice)
|
||||||
|
raw = _make(c, preset="hub")["token"]
|
||||||
|
assert _refused(_bearer(raw)) == 200
|
||||||
|
assert _refused(_bearer("sfl_short")) == 401
|
||||||
|
assert _refused(_bearer(apitokens.mint())) == 401 # well formed, never made
|
||||||
|
|
||||||
|
expiring = _make(c, preset="hub", expires_days=1)
|
||||||
|
token = db.get(ApiToken, expiring["id"])
|
||||||
|
token.expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
|
||||||
|
db.commit()
|
||||||
|
assert _refused(_bearer(expiring["token"])) == 401
|
||||||
|
assert c.get("/api/tokens").json()[1]["expired"] is True
|
||||||
|
|
||||||
|
assert c.delete(f"/api/tokens/{expiring['id']}").json() == {
|
||||||
|
"ok": True,
|
||||||
|
"deleted": expiring["id"],
|
||||||
|
}
|
||||||
|
assert (
|
||||||
|
db.execute(select(ApiTokenGrant).where(ApiTokenGrant.token_id == expiring["id"])).all()
|
||||||
|
== []
|
||||||
|
)
|
||||||
|
assert len(_audits(db, "api_token_revoke")) == 1
|
||||||
|
revoked = _make(c, preset="hub")
|
||||||
|
assert c.delete(f"/api/tokens/{revoked['id']}").status_code == 200
|
||||||
|
assert _refused(_bearer(revoked["token"])) == 401
|
||||||
|
|
||||||
|
alice.is_suspended = True
|
||||||
|
db.commit()
|
||||||
|
assert _refused(_bearer(raw)) == 401
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_about_to_run_out_warns_its_user_once_and_is_marked_in_the_list(
|
||||||
|
db, make_user, as_user
|
||||||
|
):
|
||||||
|
alice, bob = make_user(), make_user()
|
||||||
|
c = as_user(alice)
|
||||||
|
soon = _make(c, preset="hub", expires_days=3)
|
||||||
|
_make(c, preset="hub", expires_days=30) # outside the span
|
||||||
|
_make(c, preset="hub") # never runs out
|
||||||
|
gone = _make(c, preset="hub", expires_days=1)
|
||||||
|
db.get(ApiToken, gone["id"]).expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
|
||||||
|
_make(as_user(bob), preset="hub", expires_days=2)
|
||||||
|
db.commit()
|
||||||
|
assert [t["expiring"] for t in c.get("/api/tokens").json()] == [True, False, False, False]
|
||||||
|
|
||||||
|
assert apitokens.warn_expiring(db) == 2 # alice's and bob's; the run-out one is not "soon"
|
||||||
|
notes = db.execute(select(Notification).where(Notification.user_id == alice.id)).scalars()
|
||||||
|
[note] = notes.all()
|
||||||
|
assert note.type == "api_token_expiring"
|
||||||
|
assert note.data["token_id"] == soon["id"] and note.data["name"] == "a machine"
|
||||||
|
assert note.data["expires_at"] == soon["expires_at"]
|
||||||
|
assert apitokens.warn_expiring(db) == 0 # once per token
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_files_gc_runs_the_token_warning(db, make_user, as_user, _root):
|
||||||
|
_make(as_user(make_user()), preset="hub", expires_days=3)
|
||||||
|
assert gcmod.run_download_gc(db)["tokens_warned"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_another_users_token_cannot_be_revoked(db, make_user, as_user):
|
||||||
|
alice, bob = make_user(), make_user()
|
||||||
|
made = _make(as_user(alice), preset="hub")
|
||||||
|
assert as_user(bob).delete(f"/api/tokens/{made['id']}").status_code == 404
|
||||||
|
assert db.get(ApiToken, made["id"]) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_client_trying_tokens_is_slowed_down(db, make_user, as_user, monkeypatch):
|
||||||
|
alice = make_user()
|
||||||
|
raw = _make(as_user(alice), preset="hub")["token"]
|
||||||
|
monkeypatch.setattr(apitokens, "_bad", RateLimiter(3, 300))
|
||||||
|
for _ in range(3):
|
||||||
|
assert _refused(_bearer(apitokens.mint())) == 401
|
||||||
|
slowed = _bearer(apitokens.mint()).get("/api/files/folders")
|
||||||
|
assert slowed.status_code == 429
|
||||||
|
assert slowed.json()["detail"]["reason"] == "api_token_rate_limited"
|
||||||
|
# Even a good token from that client waits out the window: a guesser cannot tell a hit.
|
||||||
|
assert _refused(_bearer(raw)) == 429
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_runaway_machine_is_held_back_per_token(db, make_user, as_user, monkeypatch):
|
||||||
|
c = as_user(make_user())
|
||||||
|
first, second = _make(c, preset="hub")["token"], _make(c, preset="hub")["token"]
|
||||||
|
monkeypatch.setattr(apitokens, "_calls", RateLimiter(2, 60))
|
||||||
|
assert [_refused(_bearer(first)) for _ in range(3)] == [200, 200, 429]
|
||||||
|
assert _refused(_bearer(second)) == 200
|
||||||
|
|
||||||
|
|
||||||
|
# --- what a token leaves behind ------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_tokens_use_is_stamped_and_its_uploads_and_deletes_name_it_in_the_audit(
|
||||||
|
db, make_user, as_user, _root, _no_probe, upload_headers
|
||||||
|
):
|
||||||
|
alice = make_user()
|
||||||
|
made = _make(as_user(alice), preset="hub")
|
||||||
|
hub = made["grants"][0]["folder_id"]
|
||||||
|
assert db.get(ApiToken, made["id"]).last_used_at is None
|
||||||
|
m = _bearer(made["token"])
|
||||||
|
up = m.post("/api/files/upload", content=b"hi", headers=upload_headers("h.txt", folder=hub))
|
||||||
|
assert up.status_code == 200
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(ApiToken, made["id"]).last_used_at is not None
|
||||||
|
assert m.delete(f"/api/files/{up.json()['id']}").status_code == 200
|
||||||
|
via = f"api token {made['prefix']}…"
|
||||||
|
assert [a.summary for a in _audits(db, "file_upload")] == [via]
|
||||||
|
assert [a.summary for a in _audits(db, "file_delete")] == [via]
|
||||||
@@ -493,7 +493,7 @@ def _revoke_as_google_adoption(db, user) -> None:
|
|||||||
Google callback adopting an account whose email was never verified."""
|
Google callback adopting an account whose email was never verified."""
|
||||||
from app.routes.auth import revoke_untrusted_preset_password
|
from app.routes.auth import revoke_untrusted_preset_password
|
||||||
|
|
||||||
revoke_untrusted_preset_password(user)
|
revoke_untrusted_preset_password(db, user)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -221,7 +221,7 @@ def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes():
|
|||||||
CAPPED_BODY_SITES = {
|
CAPPED_BODY_SITES = {
|
||||||
("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"),
|
("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"),
|
||||||
("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"),
|
("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"),
|
||||||
("downloads.py", "spool_upload", "capped_stream_to", "cap"),
|
("files.py", "spool_upload", "capped_stream_to", "cap"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -367,7 +367,7 @@ def test_every_body_reading_site_is_known_and_fits_under_its_ceiling():
|
|||||||
assert ("POST", "/api/debug-sessions/{sid}/screenshots") in bodylimit.ROUTE_MAX_BODY_BYTES
|
assert ("POST", "/api/debug-sessions/{sid}/screenshots") in bodylimit.ROUTE_MAX_BODY_BYTES
|
||||||
# The upload site's limit is the sysconfig knob, which is bounded BY the edge ceiling rather
|
# The upload site's limit is the sysconfig knob, which is bounded BY the edge ceiling rather
|
||||||
# than being a constant to compare against — that pairing is its own test below.
|
# than being a constant to compare against — that pairing is its own test below.
|
||||||
assert ("POST", "/api/downloads/upload") in bodylimit.ROUTE_MAX_BODY_BYTES
|
assert ("POST", "/api/files/upload") in bodylimit.ROUTE_MAX_BODY_BYTES
|
||||||
|
|
||||||
|
|
||||||
def test_release_read_connection_ends_the_transaction_it_was_called_for(db):
|
def test_release_read_connection_ends_the_transaction_it_was_called_for(db):
|
||||||
@@ -467,7 +467,7 @@ def test_every_body_reading_site_hands_its_pooled_connection_back_first():
|
|||||||
assert not missing, (
|
assert not missing, (
|
||||||
"a route reads its own request body while still inside its read transaction, pinning a "
|
"a route reads its own request body while still inside its read transaction, pinning a "
|
||||||
f"pooled connection for the whole transfer: {missing}. Call "
|
f"pooled connection for the whole transfer: {missing}. Call "
|
||||||
"db.release_read_connection(db) before the await — see routes/downloads.py:spool_upload."
|
"db.release_read_connection(db) before the await — see routes/files.py:spool_upload."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -487,7 +487,7 @@ def test_the_upload_ceiling_and_the_admin_knob_cannot_drift_apart():
|
|||||||
`errors.tooLarge` exists and why the dialog checks the file against `upload_max_bytes` first.
|
`errors.tooLarge` exists and why the dialog checks the file against `upload_max_bytes` first.
|
||||||
"""
|
"""
|
||||||
spec = next(s for s in sysconfig.SPECS if s.key == "download_upload_max_bytes")
|
spec = next(s for s in sysconfig.SPECS if s.key == "download_upload_max_bytes")
|
||||||
edge = bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/downloads/upload")]
|
edge = bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/files/upload")]
|
||||||
assert spec.max == edge
|
assert spec.max == edge
|
||||||
|
|
||||||
|
|
||||||
@@ -540,7 +540,7 @@ def test_no_route_with_a_raised_ceiling_declares_a_body_parameter(method, templa
|
|||||||
"its dependencies, so FastAPI reads it before the first dependency runs and the raised "
|
"its dependencies, so FastAPI reads it before the first dependency runs and the raised "
|
||||||
"ceiling is all that stands between an anonymous caller and the whole allowance. Read the "
|
"ceiling is all that stands between an anonymous caller and the whole allowance. Read the "
|
||||||
"body yourself, from a dependency declared AFTER the auth one — see "
|
"body yourself, from a dependency declared AFTER the auth one — see "
|
||||||
"routes/downloads.py:spool_upload."
|
"routes/files.py:spool_upload."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,810 @@
|
|||||||
|
"""Ad-hoc clips (downloads/clip): the spec, the one ffmpeg command every clip goes through, the route,
|
||||||
|
and the worker path for both sources.
|
||||||
|
|
||||||
|
The ffmpeg command is judged at argv level: where the seek sits, which streams are mapped, what the
|
||||||
|
video filter does to an HDR source. The worker runs with ffmpeg / yt-dlp replaced by stand-ins that
|
||||||
|
write a file where they were told to, so what is checked is what the pipeline asks of them and what
|
||||||
|
it stores afterwards.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app import worker as workermod
|
||||||
|
from app.config import settings
|
||||||
|
from app.downloads import clip as clipmod
|
||||||
|
from app.downloads import errors as dlerrors
|
||||||
|
from app.downloads import formats, pipeline, quota, runner, service
|
||||||
|
from app.downloads import probe as probemod
|
||||||
|
from app.downloads.edit import INPUT_GUARD
|
||||||
|
from app.models import Channel, DownloadJob, MediaAsset, PlexItem, PlexLibrary, PlexShow, Video
|
||||||
|
from app.plex import paths as plex_paths
|
||||||
|
|
||||||
|
# --- the spec -----------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_spec_is_canonical_so_the_same_marks_hit_the_same_clip():
|
||||||
|
a = clipmod.normalize("youtube", "abcdefghijk", 10.00049, 40.0)
|
||||||
|
b = clipmod.normalize("youtube", "abcdefghijk", 10.0, 40.0004)
|
||||||
|
assert a == b == {"source": "youtube", "ref": "abcdefghijk", "start_s": 10.0, "end_s": 40.0}
|
||||||
|
assert clipmod.sig(a) == clipmod.sig(b)
|
||||||
|
assert clipmod.sig(a) != clipmod.sig({**a, "end_s": 41.0})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("args", "reason"),
|
||||||
|
[
|
||||||
|
(("vimeo", "abcdefghijk", 0, 10), "bad_source"),
|
||||||
|
(("youtube", "short", 0, 10), "bad_source"),
|
||||||
|
(("youtube", "abcdefghij/", 0, 10), "bad_source"),
|
||||||
|
(("plex", "12a", 0, 10), "bad_source"),
|
||||||
|
(("youtube", "abcdefghijk", -1, 10), "bad_range"),
|
||||||
|
(("youtube", "abcdefghijk", 0, float("nan")), "bad_range"),
|
||||||
|
(("youtube", "abcdefghijk", 5, 5.5), "too_short"),
|
||||||
|
(("youtube", "abcdefghijk", 30, 10), "too_short"),
|
||||||
|
(("youtube", "abcdefghijk", 0, 600.001), "too_long"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_a_range_it_cannot_cut_is_refused_with_its_reason(args, reason):
|
||||||
|
with pytest.raises(clipmod.ClipError) as e:
|
||||||
|
clipmod.normalize(*args)
|
||||||
|
assert e.value.reason == reason
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_ten_minute_cap_itself_is_allowed():
|
||||||
|
assert clipmod.duration(clipmod.normalize("plex", "42", 0, 600)) == 600
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_audio_track_is_kept_for_plex_only_and_must_be_sane():
|
||||||
|
assert clipmod.normalize("plex", "42", 0, 10, 2)["audio_track"] == 2
|
||||||
|
assert "audio_track" not in clipmod.normalize("youtube", "abcdefghijk", 0, 10, 2)
|
||||||
|
with pytest.raises(clipmod.ClipError, match="bad_audio"):
|
||||||
|
clipmod.normalize("plex", "42", 0, 10, -1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_youtube_clip_links_its_video_at_the_clip_start_and_a_plex_clip_links_nothing():
|
||||||
|
yt = clipmod.normalize("youtube", "abcdefghijk", 75.9, 90)
|
||||||
|
assert clipmod.reference_url(yt) == "https://www.youtube.com/watch?v=abcdefghijk&t=75s"
|
||||||
|
assert clipmod.reference_url(clipmod.normalize("plex", "42", 0, 10)) is None
|
||||||
|
|
||||||
|
|
||||||
|
# --- the ffmpeg command ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _cmd(**kw):
|
||||||
|
args = {"audio_track": None, "hdr": False, "threads": 3, **kw}
|
||||||
|
return clipmod.build_cmd(Path("/m/in.mkv"), Path("/s/out.mp4"), 61.5, 30.0, **args)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_seek_is_on_the_input_so_the_clip_starts_on_the_marked_frame():
|
||||||
|
cmd = _cmd()
|
||||||
|
i = cmd.index("-i")
|
||||||
|
assert cmd[cmd.index("-ss") + 1] == "61.500" and cmd.index("-ss") < i
|
||||||
|
assert cmd[i + 1] == "/m/in.mkv"
|
||||||
|
assert cmd[cmd.index("-t") + 1] == "30.000" and cmd.index("-t") > i
|
||||||
|
# The single-file demuxer allowlist goes with every library read.
|
||||||
|
g = cmd.index(INPUT_GUARD[0])
|
||||||
|
assert cmd[g : g + len(INPUT_GUARD)] == INPUT_GUARD and g < i
|
||||||
|
assert cmd[-1] == "/s/out.mp4"
|
||||||
|
|
||||||
|
|
||||||
|
def test_it_maps_the_first_video_and_the_heard_audio_track_and_nothing_else():
|
||||||
|
cmd = _cmd(audio_track=2)
|
||||||
|
maps = [cmd[i + 1] for i, a in enumerate(cmd) if a == "-map"]
|
||||||
|
assert maps == ["0:v:0", "0:a:2"]
|
||||||
|
assert "-sn" in cmd and "-dn" in cmd
|
||||||
|
# An mkv's chapters would ride into the .mp4 as a data track of their own (measured on a Blu-ray
|
||||||
|
# rip: `bin_data`), which -dn does not drop.
|
||||||
|
assert cmd[cmd.index("-map_chapters") + 1] == "-1"
|
||||||
|
# No track chosen: the first one, and a source with no audio at all still cuts.
|
||||||
|
maps = [c for i, c in enumerate(_cmd()) if i and _cmd()[i - 1] == "-map"]
|
||||||
|
assert maps == ["0:v:0", "0:a:0?"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_output_is_phone_safe_stereo_h264_at_most_1080p():
|
||||||
|
cmd = _cmd()
|
||||||
|
assert cmd[cmd.index("-c:v") + 1] == "libx264"
|
||||||
|
assert cmd[cmd.index("-pix_fmt") + 1] == "yuv420p"
|
||||||
|
assert cmd[cmd.index("-c:a") + 1] == "aac" and cmd[cmd.index("-ac") + 1] == "2"
|
||||||
|
assert cmd[cmd.index("-threads") + 1] == "3"
|
||||||
|
vf = cmd[cmd.index("-vf") + 1]
|
||||||
|
assert "min(1080,ih)" in vf and "tonemap" not in vf
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_hdr_source_is_tonemapped_to_sdr_before_it_is_scaled():
|
||||||
|
vf = _cmd(hdr=True)[_cmd(hdr=True).index("-vf") + 1]
|
||||||
|
assert vf.index("tonemap") < vf.index("zscale=t=bt709") < vf.index("min(1080,ih)")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("transfer", "hdr"),
|
||||||
|
[("smpte2084", True), ("arib-std-b67", True), ("bt709", False), (None, False)],
|
||||||
|
)
|
||||||
|
def test_hdr_is_read_from_the_first_video_stream(transfer, hdr):
|
||||||
|
data = {
|
||||||
|
"streams": [{"codec_type": "audio"}, {"codec_type": "video", "color_transfer": transfer}]
|
||||||
|
}
|
||||||
|
assert clipmod.is_hdr(data) is hdr
|
||||||
|
|
||||||
|
|
||||||
|
# --- the crop ----------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_FRAME = {"x": 0.5, "y": 0.25, "w": 0.25, "h": 0.5, "aspect": 16 / 9}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_crop_frame_is_canonical_and_a_frame_over_the_whole_box_is_no_crop():
|
||||||
|
a = clipmod.normalize("plex", "42", 0, 10, crop={**_FRAME, "x": 0.50004})
|
||||||
|
b = clipmod.normalize("plex", "42", 0, 10, crop=_FRAME)
|
||||||
|
assert a["crop"] == b["crop"] == {"x": 0.5, "y": 0.25, "w": 0.25, "h": 0.5, "aspect": 1.7778}
|
||||||
|
assert clipmod.sig(a) == clipmod.sig(b)
|
||||||
|
whole = clipmod.normalize(
|
||||||
|
"plex", "42", 0, 10, crop={"x": 0, "y": 0, "w": 1, "h": 1, "aspect": 1}
|
||||||
|
)
|
||||||
|
assert whole == clipmod.normalize("plex", "42", 0, 10)
|
||||||
|
# A far edge a hair past the box (the client's rounding) is the box's edge.
|
||||||
|
assert clipmod.normalize("plex", "42", 0, 10, crop={**_FRAME, "w": 0.5004})["crop"]["w"] == 0.5
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"frame",
|
||||||
|
[
|
||||||
|
{**_FRAME, "w": 0.05},
|
||||||
|
{**_FRAME, "h": 0.09},
|
||||||
|
{**_FRAME, "x": -0.1},
|
||||||
|
{**_FRAME, "x": 0.8},
|
||||||
|
{**_FRAME, "y": 0.6},
|
||||||
|
{**_FRAME, "aspect": 0},
|
||||||
|
{**_FRAME, "aspect": 9},
|
||||||
|
{**_FRAME, "x": float("nan")},
|
||||||
|
{"x": 0.1, "y": 0.1, "w": 0.5},
|
||||||
|
{**_FRAME, "h": "half"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_a_crop_frame_it_cannot_cut_is_refused(frame):
|
||||||
|
with pytest.raises(clipmod.ClipError) as e:
|
||||||
|
clipmod.normalize("plex", "42", 0, 10, crop=frame)
|
||||||
|
assert e.value.reason == "bad_crop"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("stream", "dar"),
|
||||||
|
[
|
||||||
|
({"width": 1920, "height": 1080}, 16 / 9),
|
||||||
|
# An anamorphic DVD: 720×480 stored, shown 16:9.
|
||||||
|
({"width": 720, "height": 480, "sample_aspect_ratio": "32:27"}, 16 / 9),
|
||||||
|
({"width": 640, "height": 480, "sample_aspect_ratio": "0:1"}, 4 / 3),
|
||||||
|
# A phone video stored landscape, shown upright.
|
||||||
|
({"width": 1280, "height": 720, "side_data_list": [{"rotation": -90}]}, 9 / 16),
|
||||||
|
({"width": 1280, "height": 720, "tags": {"rotate": "270"}}, 9 / 16),
|
||||||
|
({"width": 1280, "height": 720, "side_data_list": [{"rotation": 180}]}, 16 / 9),
|
||||||
|
({"width": None, "height": 720}, None),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_the_display_aspect_is_the_picture_as_a_player_shows_it(stream, dar):
|
||||||
|
got = clipmod.display_aspect(
|
||||||
|
{"streams": [{"codec_type": "audio"}, {"codec_type": "video", **stream}]}
|
||||||
|
)
|
||||||
|
assert got == pytest.approx(dar) if dar else got is None
|
||||||
|
|
||||||
|
|
||||||
|
def _crop_args(f: str) -> list[float]:
|
||||||
|
# crop=trunc(iw*W/2)*2:trunc(ih*H/2)*2:trunc(iw*X/2)*2:trunc(ih*Y/2)*2 → [W, H, X, Y]
|
||||||
|
assert f.startswith("crop=")
|
||||||
|
return [float(p.split("*")[1].split("/")[0]) for p in f[5:].split(":")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_frame_on_a_picture_that_fills_its_box_is_cut_as_drawn():
|
||||||
|
assert _crop_args(clipmod.crop_filter(_FRAME, 16 / 9)) == pytest.approx([0.25, 0.5, 0.5, 0.25])
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_frame_drawn_in_a_16_9_box_over_a_4_3_picture_maps_onto_the_picture():
|
||||||
|
# The 4:3 picture fills 3/4 of the box's width from 1/8 in: a frame over its right half's middle.
|
||||||
|
frame = {"x": 0.5, "y": 0.25, "w": 0.1875, "h": 0.5, "aspect": 16 / 9}
|
||||||
|
assert _crop_args(clipmod.crop_filter(frame, 4 / 3)) == pytest.approx([0.25, 0.5, 0.5, 0.25])
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_part_of_a_frame_over_a_bar_is_dropped_and_a_frame_on_a_bar_alone_is_no_cut():
|
||||||
|
over = {"x": 0.0, "y": 0.0, "w": 0.25, "h": 0.5, "aspect": 16 / 9}
|
||||||
|
w, h, x, y = _crop_args(clipmod.crop_filter(over, 4 / 3))
|
||||||
|
assert (w, h, x, y) == pytest.approx([1 / 6, 0.5, 0, 0], abs=1e-6)
|
||||||
|
assert clipmod.crop_filter({**over, "w": 0.13}, 4 / 3) is None
|
||||||
|
# A letterboxed film: the frame over the whole box is the whole picture.
|
||||||
|
whole = {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0, "aspect": 16 / 9}
|
||||||
|
assert clipmod.crop_filter(whole, 2.39) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_crop_comes_first_so_the_tonemap_and_the_scale_work_on_what_it_keeps():
|
||||||
|
f = clipmod.crop_filter(_FRAME, 16 / 9)
|
||||||
|
vf = _cmd(hdr=True, crop=f)[_cmd(hdr=True, crop=f).index("-vf") + 1]
|
||||||
|
assert vf.startswith(f + ",") and vf.index("tonemap") < vf.index("min(1080,ih)")
|
||||||
|
assert "crop" not in _cmd()[_cmd().index("-vf") + 1]
|
||||||
|
|
||||||
|
|
||||||
|
# --- the route ---------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def yt_video(db):
|
||||||
|
db.add(Channel(id="UC_clip", title="The Channel"))
|
||||||
|
db.flush()
|
||||||
|
db.add(
|
||||||
|
Video(
|
||||||
|
id="abcdefghijk",
|
||||||
|
channel_id="UC_clip",
|
||||||
|
title="A Funny Video",
|
||||||
|
duration_seconds=300,
|
||||||
|
thumbnail_url="https://i.ytimg.com/vi/abcdefghijk/hq.jpg",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return "abcdefghijk"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def plex_episode(db, tmp_path, monkeypatch):
|
||||||
|
lib = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||||
|
db.add(lib)
|
||||||
|
db.flush()
|
||||||
|
show = PlexShow(rating_key="s9", library_id=lib.id, title="The Show")
|
||||||
|
db.add(show)
|
||||||
|
db.flush()
|
||||||
|
db.add(
|
||||||
|
PlexItem(
|
||||||
|
rating_key="777",
|
||||||
|
library_id=lib.id,
|
||||||
|
kind="episode",
|
||||||
|
title="Pilot",
|
||||||
|
show_id=show.id,
|
||||||
|
season_number=1,
|
||||||
|
episode_number=2,
|
||||||
|
duration_s=1800,
|
||||||
|
file_path="/data/shows/pilot.mkv",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
media = tmp_path / "pilot.mkv"
|
||||||
|
media.write_bytes(b"m" * 8)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
plex_paths,
|
||||||
|
"local_media_path",
|
||||||
|
lambda _db, p: media if p == "/data/shows/pilot.mkv" else None,
|
||||||
|
)
|
||||||
|
return media
|
||||||
|
|
||||||
|
|
||||||
|
def _clip(client, **body):
|
||||||
|
return client.post("/api/downloads/clip", json={"start_s": 10, "end_s": 40, **body})
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_youtube_clip_is_queued_as_its_own_job_titled_after_its_video(
|
||||||
|
db, make_user, as_user, yt_video
|
||||||
|
):
|
||||||
|
r = _clip(as_user(make_user()), source="youtube", ref=yt_video)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
job = r.json()
|
||||||
|
assert (job["job_kind"], job["status"], job["display_name"]) == (
|
||||||
|
"clip",
|
||||||
|
"queued",
|
||||||
|
"A Funny Video (clip)",
|
||||||
|
)
|
||||||
|
assert job["edit_spec"] == {"source": "youtube", "ref": yt_video, "start_s": 10, "end_s": 40}
|
||||||
|
assert job["source_url"] == f"https://www.youtube.com/watch?v={yt_video}&t=10s"
|
||||||
|
asset = db.get(MediaAsset, job["asset"]["id"])
|
||||||
|
assert (asset.duration_s, asset.uploader, asset.media_kind) == (30, "The Channel", "video")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_same_range_again_is_the_same_clip_but_never_another_users(
|
||||||
|
db, make_user, as_user, yt_video
|
||||||
|
):
|
||||||
|
a, b = make_user(), make_user()
|
||||||
|
first = _clip(as_user(a), source="youtube", ref=yt_video).json()["asset"]["id"]
|
||||||
|
again = _clip(as_user(a), source="youtube", ref=yt_video).json()["asset"]["id"]
|
||||||
|
other = _clip(as_user(b), source="youtube", ref=yt_video).json()["asset"]["id"]
|
||||||
|
longer = _clip(as_user(a), source="youtube", ref=yt_video, end_s=41).json()["asset"]["id"]
|
||||||
|
assert first == again
|
||||||
|
assert len({first, other, longer}) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_ready_clip_asked_for_again_is_done_at_once(db, make_user, as_user, yt_video):
|
||||||
|
u = make_user()
|
||||||
|
first = _clip(as_user(u), source="youtube", ref=yt_video).json()
|
||||||
|
asset = db.get(MediaAsset, first["asset"]["id"])
|
||||||
|
asset.status, asset.rel_path = "ready", ".edits/1/clip_1.mp4"
|
||||||
|
db.commit()
|
||||||
|
again = _clip(as_user(u), source="youtube", ref=yt_video).json()
|
||||||
|
assert (again["status"], again["progress"]) == ("done", 100)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("body", "status", "reason", "params"),
|
||||||
|
[
|
||||||
|
# The cap travels with its refusal, so the client never states a stale one.
|
||||||
|
({"end_s": 700}, 422, "too_long", {"limit_s": clipmod.MAX_CLIP_S}),
|
||||||
|
({"start_s": 299.5, "end_s": 330}, 422, "past_end", {}),
|
||||||
|
({"ref": "nope"}, 422, "bad_source", {}),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_a_youtube_clip_it_cannot_cut_says_why(
|
||||||
|
db, make_user, as_user, yt_video, body, status, reason, params
|
||||||
|
):
|
||||||
|
r = _clip(as_user(make_user()), **{"source": "youtube", "ref": yt_video, **body})
|
||||||
|
assert r.status_code == status
|
||||||
|
assert r.json()["detail"] == {"code": "clip", "reason": reason, **params}
|
||||||
|
assert db.query(DownloadJob).count() == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_plex_clip_is_titled_show_episode_title_and_keeps_its_audio_track(
|
||||||
|
db, make_user, as_user, plex_episode
|
||||||
|
):
|
||||||
|
r = _clip(as_user(make_user()), source="plex", ref="777", audio_track=1)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
job = r.json()
|
||||||
|
assert job["display_name"] == "The Show — S1·E2 — Pilot (clip)"
|
||||||
|
assert job["edit_spec"]["audio_track"] == 1
|
||||||
|
assert job["source_url"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_plex_clip_of_an_unknown_item_is_a_404_and_of_an_unreachable_file_a_422(
|
||||||
|
db, make_user, as_user, plex_episode
|
||||||
|
):
|
||||||
|
c = as_user(make_user())
|
||||||
|
assert _clip(c, source="plex", ref="999").json()["detail"]["reason"] == "not_found"
|
||||||
|
assert _clip(c, source="plex", ref="999").status_code == 404
|
||||||
|
item = db.query(PlexItem).one()
|
||||||
|
item.file_path = "/data/elsewhere.mkv"
|
||||||
|
db.commit()
|
||||||
|
r = _clip(c, source="plex", ref="777")
|
||||||
|
assert (r.status_code, r.json()["detail"]["reason"]) == (422, "media_missing")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_clip_over_quota_is_refused_like_a_download(
|
||||||
|
db, make_user, as_user, yt_video, monkeypatch
|
||||||
|
):
|
||||||
|
def full(_db, _uid):
|
||||||
|
raise quota.QuotaExceeded("max_jobs", limit=5, current=5)
|
||||||
|
|
||||||
|
monkeypatch.setattr(quota, "check_enqueue", full)
|
||||||
|
r = _clip(as_user(make_user()), source="youtube", ref=yt_video)
|
||||||
|
assert r.status_code == 422
|
||||||
|
assert db.query(DownloadJob).count() == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_clip_is_not_redownloadable(db, make_user, as_user, yt_video):
|
||||||
|
c = as_user(make_user())
|
||||||
|
job = _clip(c, source="youtube", ref=yt_video).json()
|
||||||
|
row = db.get(DownloadJob, job["id"])
|
||||||
|
row.status, row.ready_at = "done", row.created_at
|
||||||
|
db.commit()
|
||||||
|
r = c.post(f"/api/downloads/{job['id']}/redownload")
|
||||||
|
assert (r.status_code, r.json()["detail"]["reason"]) == (409, "not_redownloadable")
|
||||||
|
|
||||||
|
|
||||||
|
# --- the worker ----------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_clip_job_reaches_the_clip_entry(monkeypatch):
|
||||||
|
called = []
|
||||||
|
for name in ("_convert_upload", "_process_edit", "_download", "_process_clip"):
|
||||||
|
monkeypatch.setattr(pipeline, name, lambda *a, _n=name: called.append(_n))
|
||||||
|
runner._run(5, 7, "clip", "clip", "1:youtube:abcdefghijk", {}, 1)
|
||||||
|
assert called == ["_process_clip"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def worker(db, tmp_path, monkeypatch):
|
||||||
|
"""The worker path with its external programs replaced: ffmpeg writes the file it was given, the
|
||||||
|
probe answers what the test says, the phone-safe step and the poster pass through."""
|
||||||
|
monkeypatch.setattr(settings, "files_root", str(tmp_path / "root"))
|
||||||
|
seen: dict = {}
|
||||||
|
|
||||||
|
def ffmpeg(cmd, *_a, **_k):
|
||||||
|
seen["cmd"] = cmd
|
||||||
|
Path(cmd[-1]).write_bytes(b"clip")
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", ffmpeg)
|
||||||
|
monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, p, *_a, **_k: p)
|
||||||
|
monkeypatch.setattr(pipeline.editmod, "ensure_poster", lambda *_a, **_k: None)
|
||||||
|
monkeypatch.setattr(probemod, "probe_media", lambda _p: ("h264", "aac", 1920, 1080, 30))
|
||||||
|
seen["probe"] = {"streams": [{"codec_type": "video"}, {"codec_type": "audio"}]}
|
||||||
|
monkeypatch.setattr(probemod, "probe_json", lambda _p: seen["probe"])
|
||||||
|
|
||||||
|
def run(job_json):
|
||||||
|
job = db.get(DownloadJob, job_json["id"])
|
||||||
|
asset = db.get(MediaAsset, job.asset_id)
|
||||||
|
job.status, asset.status = "running", "downloading"
|
||||||
|
db.commit()
|
||||||
|
pipeline._process_clip(job.id, asset.id)
|
||||||
|
db.expire_all()
|
||||||
|
return db.get(DownloadJob, job.id), db.get(MediaAsset, asset.id)
|
||||||
|
|
||||||
|
seen["run"] = run
|
||||||
|
return seen
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_plex_clip_is_cut_from_the_library_file_with_its_audio_track(
|
||||||
|
db, make_user, as_user, plex_episode, worker
|
||||||
|
):
|
||||||
|
worker["probe"] = {
|
||||||
|
"streams": [
|
||||||
|
{"codec_type": "video", "color_transfer": "smpte2084"},
|
||||||
|
{"codec_type": "audio"},
|
||||||
|
{"codec_type": "audio"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
job = _clip(as_user(make_user()), source="plex", ref="777", audio_track=1).json()
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
assert (job.status, asset.status, asset.container, asset.height) == (
|
||||||
|
"done",
|
||||||
|
"ready",
|
||||||
|
"mp4",
|
||||||
|
1080,
|
||||||
|
)
|
||||||
|
cmd = worker["cmd"]
|
||||||
|
assert cmd[cmd.index("-i") + 1] == str(plex_episode)
|
||||||
|
assert cmd[cmd.index("-ss") + 1] == "10.000"
|
||||||
|
assert "0:a:1" in cmd and "tonemap" in cmd[cmd.index("-vf") + 1]
|
||||||
|
assert asset.rel_path.startswith(".edits/") and asset.rel_path.endswith(".mp4")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_cropped_plex_clip_cuts_the_frame_from_the_picture_and_is_its_own_clip(
|
||||||
|
db, make_user, as_user, plex_episode, worker
|
||||||
|
):
|
||||||
|
worker["probe"] = {
|
||||||
|
"streams": [
|
||||||
|
{"codec_type": "video", "width": 720, "height": 480, "sample_aspect_ratio": "32:27"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
c = as_user(make_user())
|
||||||
|
plain = _clip(c, source="plex", ref="777").json()
|
||||||
|
job = _clip(c, source="plex", ref="777", crop=_FRAME).json()
|
||||||
|
assert job["asset"]["id"] != plain["asset"]["id"]
|
||||||
|
assert job["edit_spec"]["crop"]["w"] == 0.25
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
assert (job.status, asset.status) == ("done", "ready")
|
||||||
|
vf = worker["cmd"][worker["cmd"].index("-vf") + 1]
|
||||||
|
# Stored to four decimals, the box's 16:9 is a hair off the file's: well under a pixel.
|
||||||
|
assert _crop_args(vf.split(",")[0]) == pytest.approx([0.25, 0.5, 0.5, 0.25], abs=1e-4)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_crop_frame_that_misses_the_picture_fails_and_says_so(
|
||||||
|
db, make_user, as_user, plex_episode, worker
|
||||||
|
):
|
||||||
|
# A 4:3 file, a frame drawn over its pillar bar in a 16:9 box.
|
||||||
|
worker["probe"] = {"streams": [{"codec_type": "video", "width": 640, "height": 480}]}
|
||||||
|
frame = {"x": 0.0, "y": 0.0, "w": 0.12, "h": 0.5, "aspect": 16 / 9}
|
||||||
|
job = _clip(as_user(make_user()), source="plex", ref="777", crop=frame).json()
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
assert (job.status, asset.status) == ("error", "error")
|
||||||
|
assert "crop" in job.error
|
||||||
|
assert "cmd" not in worker
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_crop_frame_the_route_cannot_cut_says_why(db, make_user, as_user, yt_video):
|
||||||
|
r = _clip(as_user(make_user()), source="youtube", ref=yt_video, crop={**_FRAME, "w": 0.01})
|
||||||
|
assert r.status_code == 422
|
||||||
|
assert r.json()["detail"] == {"code": "clip", "reason": "bad_crop"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_plex_clip_of_a_track_the_file_does_not_have_fails_and_says_so(
|
||||||
|
db, make_user, as_user, plex_episode, worker
|
||||||
|
):
|
||||||
|
job = _clip(as_user(make_user()), source="plex", ref="777", audio_track=3).json()
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
assert (job.status, asset.status) == ("error", "error")
|
||||||
|
assert "audio track" in job.error
|
||||||
|
assert "cmd" not in worker
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_youtube_clip_fetches_only_its_range_and_cuts_it_from_zero(
|
||||||
|
db, make_user, as_user, yt_video, worker, monkeypatch
|
||||||
|
):
|
||||||
|
asked: dict = {}
|
||||||
|
|
||||||
|
def extract(job_id, asset_id, url, spec, staging, adapter, **kw):
|
||||||
|
asked.update(url=url, spec=spec, **kw)
|
||||||
|
f = Path(staging) / "abcdefghijk.mp4"
|
||||||
|
f.write_bytes(b"range")
|
||||||
|
return {"requested_downloads": [{"filepath": str(f)}]}
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||||
|
job = _clip(as_user(make_user()), source="youtube", ref=yt_video).json()
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
assert (job.status, asset.status) == ("done", "ready")
|
||||||
|
assert asked["section"] == (10, 40)
|
||||||
|
assert asked["spec"]["max_height"] == 1080
|
||||||
|
assert asked["url"].endswith(yt_video)
|
||||||
|
cmd = worker["cmd"]
|
||||||
|
assert cmd[cmd.index("-ss") + 1] == "0.000" and cmd[cmd.index("-t") + 1] == "30.000"
|
||||||
|
assert cmd[cmd.index("-i") + 1].endswith("abcdefghijk.mp4")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_youtube_clip_hands_the_range_to_yt_dlp(monkeypatch, tmp_path):
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
class FakeYDL:
|
||||||
|
def __init__(self, opts):
|
||||||
|
seen.update(opts)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_a):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def extract_info(self, *_a, **_k):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline.yt_dlp, "YoutubeDL", FakeYDL)
|
||||||
|
monkeypatch.setattr(pipeline, "_touch_job", lambda *_a: None)
|
||||||
|
adapter = pipeline.adapters.resolve_adapter("youtube", "abcdefghijk")
|
||||||
|
pipeline._extract_with_fallback(
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
"https://www.youtube.com/watch?v=abcdefghijk",
|
||||||
|
pipeline.formats.normalize({}),
|
||||||
|
tmp_path,
|
||||||
|
adapter,
|
||||||
|
section=(10.0, 40.0),
|
||||||
|
)
|
||||||
|
ranges = list(seen["download_ranges"]({}, None))
|
||||||
|
assert [(r["start_time"], r["end_time"]) for r in ranges] == [(10.0, 40.0)]
|
||||||
|
assert seen["force_keyframes_at_cuts"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_reference_url_follows_the_clip_spec(db, make_user, as_user, yt_video):
|
||||||
|
job = _clip(as_user(make_user()), source="youtube", ref=yt_video, start_s=65, end_s=90).json()
|
||||||
|
row = db.get(DownloadJob, job["id"])
|
||||||
|
assert service.reference_url(row, None) == f"https://www.youtube.com/watch?v={yt_video}&t=65s"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_youtube_clip_outside_the_catalog_takes_its_title_from_the_fetch(
|
||||||
|
db, make_user, as_user, worker, monkeypatch
|
||||||
|
):
|
||||||
|
def extract(job_id, asset_id, url, spec, staging, adapter, **kw):
|
||||||
|
f = Path(staging) / "zyxwvutsrqp.mp4"
|
||||||
|
f.write_bytes(b"range")
|
||||||
|
return {"requested_downloads": [{"filepath": str(f)}], "title": "Found On YouTube"}
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||||
|
job = _clip(as_user(make_user()), source="youtube", ref="zyxwvutsrqp").json()
|
||||||
|
assert job["display_name"] == "Clip (clip)"
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
assert (job.status, asset.title, job.display_name) == (
|
||||||
|
"done",
|
||||||
|
"Found On YouTube (clip)",
|
||||||
|
"Found On YouTube (clip)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- a YouTube clip goes with the user's own cookie, and its failure is diagnosed -----------------
|
||||||
|
|
||||||
|
_YT_COOKIES = "# Netscape HTTP Cookie File\n.youtube.com\tTRUE\t/\tTRUE\t0\tSID\tabc\n"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fernet(monkeypatch):
|
||||||
|
from cryptography.fernet import Fernet
|
||||||
|
|
||||||
|
from app import security
|
||||||
|
|
||||||
|
monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key()))
|
||||||
|
|
||||||
|
|
||||||
|
def _capture_extract(monkeypatch, asked: dict):
|
||||||
|
def extract(job_id, asset_id, url, spec, staging, adapter, **kw):
|
||||||
|
cookiefile = kw.get("cookiefile")
|
||||||
|
asked["cookiefile"] = cookiefile
|
||||||
|
asked["cookie_text"] = Path(cookiefile).read_text() if cookiefile else None
|
||||||
|
f = Path(staging) / "abcdefghijk.mp4"
|
||||||
|
f.write_bytes(b"range")
|
||||||
|
return {"requested_downloads": [{"filepath": str(f)}]}
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_youtube_clip_goes_signed_in_with_the_users_own_cookie_and_leaves_none_behind(
|
||||||
|
db, make_user, as_user, yt_video, worker, monkeypatch, fernet
|
||||||
|
):
|
||||||
|
from app.downloads import credentials
|
||||||
|
|
||||||
|
user = make_user()
|
||||||
|
credentials.upsert_credential(db, user.id, "youtube", _YT_COOKIES)
|
||||||
|
db.commit()
|
||||||
|
asked: dict = {}
|
||||||
|
_capture_extract(monkeypatch, asked)
|
||||||
|
job, _asset = worker["run"](_clip(as_user(user), source="youtube", ref=yt_video).json())
|
||||||
|
assert job.status == "done"
|
||||||
|
assert asked["cookie_text"] == _YT_COOKIES
|
||||||
|
assert not Path(asked["cookiefile"]).exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_youtube_clip_never_goes_with_another_users_cookie(
|
||||||
|
db, make_user, as_user, yt_video, worker, monkeypatch, fernet
|
||||||
|
):
|
||||||
|
from app.downloads import credentials
|
||||||
|
|
||||||
|
other = make_user()
|
||||||
|
credentials.upsert_credential(db, other.id, "youtube", _YT_COOKIES)
|
||||||
|
db.commit()
|
||||||
|
asked: dict = {}
|
||||||
|
_capture_extract(monkeypatch, asked)
|
||||||
|
job, _asset = worker["run"](_clip(as_user(make_user()), source="youtube", ref=yt_video).json())
|
||||||
|
assert job.status == "done" and asked["cookiefile"] is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("with_cookie", "detail"), [(False, "youtube.bot_wall"), (True, "youtube.cookie_rejected")]
|
||||||
|
)
|
||||||
|
def test_a_youtube_clip_on_the_bot_wall_is_diagnosed_like_a_download(
|
||||||
|
db, make_user, as_user, yt_video, worker, monkeypatch, fernet, with_cookie, detail
|
||||||
|
):
|
||||||
|
from app.downloads import credentials
|
||||||
|
|
||||||
|
user = make_user()
|
||||||
|
if with_cookie:
|
||||||
|
credentials.upsert_credential(db, user.id, "youtube", _YT_COOKIES)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
def walled(*_a, **_k):
|
||||||
|
raise RuntimeError(f"ERROR: [youtube] {yt_video}: Sign in to confirm you’re not a bot.")
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline, "_extract_with_fallback", walled)
|
||||||
|
job, asset = worker["run"](_clip(as_user(user), source="youtube", ref=yt_video).json())
|
||||||
|
assert (job.status, asset.status) == ("error", "error")
|
||||||
|
assert (job.error_code, job.error_detail) == ("login_required", detail)
|
||||||
|
|
||||||
|
|
||||||
|
# --- held to the owner's storage cap, and a clip that cannot or need not go on ---------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_range(monkeypatch):
|
||||||
|
def extract(job_id, asset_id, url, spec, staging, adapter, **kw):
|
||||||
|
f = Path(staging) / "abcdefghijk.mp4"
|
||||||
|
f.write_bytes(b"range")
|
||||||
|
return {"requested_downloads": [{"filepath": str(f)}]}
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("source", ["youtube", "plex"])
|
||||||
|
def test_a_clip_for_an_owner_whose_storage_is_full_is_refused_before_it_is_cut(
|
||||||
|
db, make_user, as_user, yt_video, plex_episode, worker, monkeypatch, source
|
||||||
|
):
|
||||||
|
_fetch_range(monkeypatch)
|
||||||
|
ref = yt_video if source == "youtube" else "777"
|
||||||
|
job = _clip(as_user(make_user()), source=source, ref=ref).json()
|
||||||
|
monkeypatch.setattr(quota, "remaining_bytes", lambda _db, _uid: 0)
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
# Refused like a download at its cap: the job says why, the asset waits for a Retry.
|
||||||
|
assert (job.status, job.error, asset.status) == (
|
||||||
|
"error",
|
||||||
|
dlerrors.QUOTA_FULL_MESSAGE,
|
||||||
|
"pending",
|
||||||
|
)
|
||||||
|
assert "cmd" not in worker
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("source", ["youtube", "plex"])
|
||||||
|
def test_a_clip_larger_than_what_its_owner_has_left_is_refused_and_not_placed(
|
||||||
|
db, make_user, as_user, yt_video, plex_episode, worker, monkeypatch, source
|
||||||
|
):
|
||||||
|
# The range fetch's own ceiling never reaches yt-dlp's ffmpeg downloader, and a Plex clip
|
||||||
|
# fetches nothing: the cut file itself is held to the cap.
|
||||||
|
_fetch_range(monkeypatch)
|
||||||
|
ref = yt_video if source == "youtube" else "777"
|
||||||
|
job = _clip(as_user(make_user()), source=source, ref=ref).json()
|
||||||
|
monkeypatch.setattr(quota, "remaining_bytes", lambda _db, _uid: 3) # the cut is 4 bytes
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
# Not "full": there is room left, just not this much.
|
||||||
|
assert (job.status, job.error, asset.status, asset.rel_path) == (
|
||||||
|
"error",
|
||||||
|
dlerrors.CLIP_TOO_BIG_MESSAGE,
|
||||||
|
"pending",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_youtube_clip_range_yt_dlp_refuses_for_its_cap_says_it_is_too_big(
|
||||||
|
db, make_user, as_user, yt_video, worker, monkeypatch
|
||||||
|
):
|
||||||
|
# Should yt-dlp's own `max_filesize` refuse the range, it returns with no file (test_formats):
|
||||||
|
# the owner's storage, not a processing fault, and nothing is cut.
|
||||||
|
def refused(*_a, max_bytes=None, **_k):
|
||||||
|
formats._YDL_LOG.debug(
|
||||||
|
f"[download] File is larger than max-filesize (9000 bytes > {max_bytes} bytes). Aborting."
|
||||||
|
)
|
||||||
|
return {"requested_downloads": [{}]}
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline, "_extract_with_fallback", refused)
|
||||||
|
job = _clip(as_user(make_user()), source="youtube", ref=yt_video).json()
|
||||||
|
monkeypatch.setattr(quota, "remaining_bytes", lambda _db, _uid: 500)
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
assert (job.status, job.error_code, job.error, asset.status) == (
|
||||||
|
"error",
|
||||||
|
"quota_exceeded",
|
||||||
|
dlerrors.CLIP_TOO_BIG_MESSAGE,
|
||||||
|
"pending",
|
||||||
|
)
|
||||||
|
assert "cmd" not in worker
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_plex_clip_of_a_silent_file_is_cut_silent(db, make_user, as_user, plex_episode, worker):
|
||||||
|
# The player names the track it plays 0 even when the file has none.
|
||||||
|
worker["probe"] = {"streams": [{"codec_type": "video"}]}
|
||||||
|
job = _clip(as_user(make_user()), source="plex", ref="777", audio_track=0).json()
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
assert (job.status, asset.status) == ("done", "ready")
|
||||||
|
assert "0:a:0?" in worker["cmd"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_plex_clip_of_a_file_the_probe_cannot_read_says_so(
|
||||||
|
db, make_user, as_user, plex_episode, worker
|
||||||
|
):
|
||||||
|
worker["probe"] = None
|
||||||
|
job = _clip(as_user(make_user()), source="plex", ref="777", audio_track=0).json()
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
assert (job.status, asset.status) == ("error", "error")
|
||||||
|
assert job.error == "The source file could not be read."
|
||||||
|
assert "cmd" not in worker
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("where", ["fetch", "encode"])
|
||||||
|
def test_a_paused_clip_frees_its_asset_for_a_later_retry(
|
||||||
|
db, make_user, as_user, yt_video, worker, monkeypatch, where
|
||||||
|
):
|
||||||
|
def stopped(*_a, **_k):
|
||||||
|
raise pipeline._Aborted() if where == "fetch" else pipeline.editmod.EditAborted()
|
||||||
|
|
||||||
|
if where == "fetch":
|
||||||
|
monkeypatch.setattr(pipeline, "_extract_with_fallback", stopped)
|
||||||
|
else:
|
||||||
|
_fetch_range(monkeypatch)
|
||||||
|
monkeypatch.setattr(pipeline.editmod, "run_ffmpeg", stopped)
|
||||||
|
job = _clip(as_user(make_user()), source="youtube", ref=yt_video).json()
|
||||||
|
job, asset = worker["run"](job)
|
||||||
|
# The route already wrote the job's own state (paused / canceled); the asset is free again.
|
||||||
|
assert (job.status, job.error, asset.status) == ("running", None, "pending")
|
||||||
|
|
||||||
|
|
||||||
|
# --- a second job on the same clip -----------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _second_job(db, client, **body):
|
||||||
|
"""Two jobs for one range: the second shares the first's asset."""
|
||||||
|
first = _clip(client, **body).json()
|
||||||
|
second = _clip(client, **body).json()
|
||||||
|
assert first["asset"]["id"] == second["asset"]["id"]
|
||||||
|
return db.get(DownloadJob, first["id"]), db.get(DownloadJob, second["id"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_second_clip_job_on_a_failed_clip_gets_the_same_youtube_diagnosis(
|
||||||
|
db, make_user, as_user, yt_video
|
||||||
|
):
|
||||||
|
_first, second = _second_job(db, as_user(make_user()), source="youtube", ref=yt_video)
|
||||||
|
asset = db.get(MediaAsset, second.asset_id)
|
||||||
|
asset.status = "error"
|
||||||
|
asset.error = f"ERROR: [youtube] {yt_video}: Sign in to confirm you’re not a bot."
|
||||||
|
second.status = "running"
|
||||||
|
db.commit()
|
||||||
|
workermod._process_job(second.id)
|
||||||
|
db.expire_all()
|
||||||
|
job = db.get(DownloadJob, second.id)
|
||||||
|
assert (job.status, job.error_code, job.error_detail) == (
|
||||||
|
"error",
|
||||||
|
"login_required",
|
||||||
|
"youtube.bot_wall",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_second_clip_job_takes_the_title_its_sibling_found(db, make_user, as_user):
|
||||||
|
_first, second = _second_job(db, as_user(make_user()), source="youtube", ref="zyxwvutsrqp")
|
||||||
|
assert second.display_name == "Clip (clip)"
|
||||||
|
asset = db.get(MediaAsset, second.asset_id)
|
||||||
|
asset.status, asset.title = "ready", "Found On YouTube (clip)"
|
||||||
|
second.status = "running"
|
||||||
|
db.commit()
|
||||||
|
workermod._finish_from_cache(second.id, asset.id)
|
||||||
|
db.expire_all()
|
||||||
|
job = db.get(DownloadJob, second.id)
|
||||||
|
assert (job.status, job.display_name) == ("done", "Found On YouTube (clip)")
|
||||||
@@ -8,6 +8,7 @@ import pytest
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
|
from app.downloads import playable
|
||||||
from app.downloads import runner
|
from app.downloads import runner
|
||||||
|
|
||||||
|
|
||||||
@@ -32,11 +33,12 @@ class TestEncodeThreadCap:
|
|||||||
assert editmod.encode_thread_cap() == 1 # max(1, 2 - 1)
|
assert editmod.encode_thread_cap() == 1 # max(1, 2 - 1)
|
||||||
|
|
||||||
|
|
||||||
class TestBrowserSafeThreads:
|
_REENCODE_VIDEO = playable.Plan("mp4", True, True, False, True)
|
||||||
|
|
||||||
|
|
||||||
|
class TestPhoneSafeThreads:
|
||||||
def test_threads_flag_added_when_capped(self):
|
def test_threads_flag_added_when_capped(self):
|
||||||
cmd = editmod.build_browser_safe_cmd(
|
cmd = playable.build_cmd("/in.mkv", "/out.mp4", _REENCODE_VIDEO, threads=2)
|
||||||
"/in.mkv", "/out.mp4", reencode_video=True, reencode_audio=False, threads=2
|
|
||||||
)
|
|
||||||
assert "-threads" in cmd
|
assert "-threads" in cmd
|
||||||
assert cmd[cmd.index("-threads") + 1] == "2"
|
assert cmd[cmd.index("-threads") + 1] == "2"
|
||||||
# The cap precedes the output file, so it binds the encoder (not input decoding).
|
# The cap precedes the output file, so it binds the encoder (not input decoding).
|
||||||
@@ -44,9 +46,7 @@ class TestBrowserSafeThreads:
|
|||||||
|
|
||||||
@pytest.mark.parametrize("threads", [0, None])
|
@pytest.mark.parametrize("threads", [0, None])
|
||||||
def test_no_threads_flag_when_uncapped(self, threads):
|
def test_no_threads_flag_when_uncapped(self, threads):
|
||||||
cmd = editmod.build_browser_safe_cmd(
|
cmd = playable.build_cmd("/in.mkv", "/out.mp4", _REENCODE_VIDEO, threads=threads)
|
||||||
"/in.mkv", "/out.mp4", reencode_video=True, reencode_audio=False, threads=threads
|
|
||||||
)
|
|
||||||
assert "-threads" not in cmd
|
assert "-threads" not in cmd
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,14 @@ from app.downloads import errors as dlerrors
|
|||||||
from app.downloads.errors import classify
|
from app.downloads.errors import classify
|
||||||
|
|
||||||
|
|
||||||
|
# Verbatim from dev job 128 (2026-10-09, yt-dlp 2026.07.04); the same link downloads under 2026.08.19.
|
||||||
|
_TIKTOK_UNEXPECTED = (
|
||||||
|
"ERROR: [TikTok] 7418778475890232609: Unexpected response from webpage request; please report "
|
||||||
|
"this issue on https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue "
|
||||||
|
"template. Confirm you are on the latest version using yt-dlp -U"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_real_ytdlp_messages_map_to_categories():
|
def test_real_ytdlp_messages_map_to_categories():
|
||||||
# Verbatim-shaped yt-dlp / ffmpeg output → the category the UI explains.
|
# Verbatim-shaped yt-dlp / ffmpeg output → the category the UI explains.
|
||||||
cases = {
|
cases = {
|
||||||
@@ -32,6 +40,7 @@ def test_real_ytdlp_messages_map_to_categories():
|
|||||||
"The source file is missing.": "source_missing",
|
"The source file is missing.": "source_missing",
|
||||||
"ERROR: unable to download video data: HTTP Error 403: Forbidden": "network",
|
"ERROR: unable to download video data: HTTP Error 403: Forbidden": "network",
|
||||||
"ERROR: Unable to download webpage: <urlopen error timed out>": "network",
|
"ERROR: Unable to download webpage: <urlopen error timed out>": "network",
|
||||||
|
_TIKTOK_UNEXPECTED: "site_changed",
|
||||||
}
|
}
|
||||||
for msg, expected in cases.items():
|
for msg, expected in cases.items():
|
||||||
assert classify(msg) == expected, f"{msg!r} → {classify(msg)!r}, expected {expected!r}"
|
assert classify(msg) == expected, f"{msg!r} → {classify(msg)!r}, expected {expected!r}"
|
||||||
@@ -63,6 +72,9 @@ def test_our_own_worker_refusals_classify_as_themselves():
|
|||||||
# fragment of the text. Pinning message and rule together is what stops an edit to the wording
|
# fragment of the text. Pinning message and rule together is what stops an edit to the wording
|
||||||
# from silently orphaning the category — the UI would then fall back to "Download failed".
|
# from silently orphaning the category — the UI would then fall back to "Download failed".
|
||||||
assert classify(dlerrors.QUOTA_FULL_MESSAGE) == "quota_exceeded"
|
assert classify(dlerrors.QUOTA_FULL_MESSAGE) == "quota_exceeded"
|
||||||
|
assert classify(dlerrors.CLIP_TOO_BIG_MESSAGE) == "quota_exceeded"
|
||||||
|
assert classify(dlerrors.CONVERTED_TOO_BIG_MESSAGE) == "quota_exceeded"
|
||||||
|
assert classify(dlerrors.DOWNLOAD_TOO_BIG_MESSAGE) == "quota_exceeded"
|
||||||
|
|
||||||
|
|
||||||
def test_the_redacted_network_text_classifies_back_to_network():
|
def test_the_redacted_network_text_classifies_back_to_network():
|
||||||
@@ -208,13 +220,25 @@ def test_our_post_without_a_video_verdict_names_the_group_link():
|
|||||||
|
|
||||||
|
|
||||||
def test_a_catalog_entry_never_leaves_its_host():
|
def test_a_catalog_entry_never_leaves_its_host():
|
||||||
# "Cannot parse data" is yt-dlp's wording for a broken extractor on every site.
|
# "Cannot parse data" is yt-dlp's wording for a broken extractor on every site: elsewhere it is
|
||||||
|
# only the category — the site changed under the extractor.
|
||||||
other = _FB_PARSE.replace("[facebook]", "[vimeo]")
|
other = _FB_PARSE.replace("[facebook]", "[vimeo]")
|
||||||
assert dlerrors.diagnose(other, url="https://vimeo.com/1") == ("unknown", None)
|
assert dlerrors.diagnose(other, url="https://vimeo.com/1") == ("site_changed", None)
|
||||||
# The Facebook text for a link that is not Facebook's (a look-alike host) is not the Facebook wall.
|
# The Facebook text for a link that is not Facebook's (a look-alike host) is not the Facebook wall.
|
||||||
assert dlerrors.diagnose(_FB_PARSE, url="https://facebook.com.evil/reel/1") == ("unknown", None)
|
assert dlerrors.diagnose(_FB_PARSE, url="https://facebook.com.evil/reel/1") == (
|
||||||
|
"site_changed",
|
||||||
|
None,
|
||||||
|
)
|
||||||
# No link known: the category only.
|
# No link known: the category only.
|
||||||
assert dlerrors.diagnose(_FB_PARSE) == ("unknown", None)
|
assert dlerrors.diagnose(_FB_PARSE) == ("site_changed", None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_bug_report_request_keeps_the_more_specific_category():
|
||||||
|
# yt-dlp asks for the bug report on these too; what they say about the media wins.
|
||||||
|
assert classify(_FB_NO_FORMATS) == "format_unavailable"
|
||||||
|
assert classify("ERROR: [x] 1: Unable to extract title; please report this issue") == (
|
||||||
|
"format_unavailable"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_diagnose_falls_back_to_the_category():
|
def test_diagnose_falls_back_to_the_category():
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ def _asset_and_job(db, user, *, sig, asset_status, job_status, error=None):
|
|||||||
|
|
||||||
|
|
||||||
def _fail_the_child(monkeypatch, tmp_path, *, cookie: bool):
|
def _fail_the_child(monkeypatch, tmp_path, *, cookie: bool):
|
||||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||||
cookiefile = tmp_path / "cookies.txt"
|
cookiefile = tmp_path / "cookies.txt"
|
||||||
cookiefile.write_text("# Netscape HTTP Cookie File\n")
|
cookiefile.write_text("# Netscape HTTP Cookie File\n")
|
||||||
@@ -230,7 +230,7 @@ def test_a_download_yt_dlp_gave_no_duration_stores_the_files_duration(db, monkey
|
|||||||
# row lost its trim action. Through the real finalize, with a real 3-second file.
|
# row lost its trim action. Through the real finalize, with a real 3-second file.
|
||||||
u = _user(db, "fb-duration@example.com")
|
u = _user(db, "fb-duration@example.com")
|
||||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||||
|
|
||||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||||
@@ -350,7 +350,7 @@ def test_a_signed_in_facebook_download_is_titled_with_its_posts_text(db, monkeyp
|
|||||||
# file name and the share preview all said "Video". The post's text is in the description.
|
# file name and the share preview all said "Video". The post's text is in the description.
|
||||||
u = _user(db, "fb-title@example.com")
|
u = _user(db, "fb-title@example.com")
|
||||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||||
|
|
||||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||||
@@ -401,7 +401,7 @@ def test_an_mp3_download_is_stored_as_an_mp3(db, monkeypatch, tmp_path):
|
|||||||
# video/mp4, which iPad's WebKit will not play. It stays an .mp3 (served as audio/mpeg).
|
# video/mp4, which iPad's WebKit will not play. It stays an .mp3 (served as audio/mpeg).
|
||||||
u = _user(db, "fb-mp3@example.com")
|
u = _user(db, "fb-mp3@example.com")
|
||||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||||
|
|
||||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||||
@@ -430,7 +430,7 @@ def test_a_video_download_is_still_made_browser_playable(db, monkeypatch, tmp_pa
|
|||||||
a, j = _asset_and_job(
|
a, j = _asset_and_job(
|
||||||
db, u, sig=f"mkv-{mode}", asset_status="downloading", job_status="running"
|
db, u, sig=f"mkv-{mode}", asset_status="downloading", job_status="running"
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||||
|
|
||||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ unlocked read: two overlapping requests for one job (a double-click, a second ta
|
|||||||
gateway timeout) both saw it holding its asset and both released the hold. The count then reaches 0
|
gateway timeout) both saw it holding its asset and both released the hold. The count then reaches 0
|
||||||
while another job still holds the asset, and GC or the ready-file delete removes it under that job.
|
while another job still holds the asset, and GC or the ready-file delete removes it under that job.
|
||||||
A pause racing a cancel did the same one step later: its unlocked write turned the released job back
|
A pause racing a cancel did the same one step later: its unlocked write turned the released job back
|
||||||
into a holder, which a later delete releases again. `_own_job_for_update` makes the second request
|
into a holder, which a later delete releases again. `own_job_for_update` makes the second request
|
||||||
wait and decide on the first one's committed result.
|
wait and decide on the first one's committed result.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -15,15 +15,12 @@ import pytest
|
|||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.apitokens import FilesCaller
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.downloads import service
|
from app.downloads import service
|
||||||
from app.models import DownloadJob, MediaAsset, User
|
from app.models import DownloadJob, MediaAsset, User
|
||||||
from app.routes.downloads import (
|
from app.routes.downloads import cancel_download, pause_download
|
||||||
_own_job_for_update,
|
from app.routes.files import delete_file, own_job_for_update
|
||||||
cancel_download,
|
|
||||||
delete_download,
|
|
||||||
pause_download,
|
|
||||||
)
|
|
||||||
|
|
||||||
_REF = "https://example.com/video"
|
_REF = "https://example.com/video"
|
||||||
|
|
||||||
@@ -54,11 +51,16 @@ def _asset_held_by_two_jobs(db, status="error"):
|
|||||||
return mine, asset, jobs[0]
|
return mine, asset, jobs[0]
|
||||||
|
|
||||||
|
|
||||||
|
def delete_file_as(job_id, *, user, db):
|
||||||
|
"""`delete_file` as the user's own request (it takes the Files caller, not the user)."""
|
||||||
|
return delete_file(job_id, caller=FilesCaller(user=user), db=db)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("status", "transition", "route", "left"),
|
("status", "transition", "route", "left"),
|
||||||
[
|
[
|
||||||
("error", service.cancel, cancel_download, "canceled"),
|
("error", service.cancel, cancel_download, "canceled"),
|
||||||
("error", service.delete, delete_download, None),
|
("error", service.delete, delete_file_as, None),
|
||||||
("queued", service.cancel, pause_download, "canceled"),
|
("queued", service.cancel, pause_download, "canceled"),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@@ -80,7 +82,7 @@ def test_overlapping_requests_for_one_job_release_its_hold_once(
|
|||||||
finished.set()
|
finished.set()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
locked = _own_job_for_update(first, user, job_id)
|
locked = own_job_for_update(first, user, job_id)
|
||||||
other = threading.Thread(target=the_other_request)
|
other = threading.Thread(target=the_other_request)
|
||||||
other.start()
|
other.start()
|
||||||
# The second request waits for the first one's locks instead of deciding on the same row.
|
# The second request waits for the first one's locks instead of deciding on the same row.
|
||||||
|
|||||||
@@ -11,7 +11,8 @@ its OWN session, so every test COMMITS its setup before calling it.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import pipeline, quota
|
from app.downloads import errors as dlerrors
|
||||||
|
from app.downloads import formats, pipeline, quota
|
||||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
|
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
|
||||||
from app.worker import _claim_job
|
from app.worker import _claim_job
|
||||||
|
|
||||||
@@ -155,6 +156,25 @@ def test_unlimited_user_is_never_excluded(db):
|
|||||||
assert _claim_job() == q.id
|
assert _claim_job() == q.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_suspended_users_queued_jobs_wait(db):
|
||||||
|
# Suspending a user blocks their downloads (quota.resolve's advice): a job they queued before
|
||||||
|
# stays queued, unclaimed, and runs once the suspension is lifted.
|
||||||
|
s = User(email="suspended@example.com", is_suspended=True)
|
||||||
|
o = User(email="other@example.com")
|
||||||
|
db.add_all([s, o])
|
||||||
|
db.flush()
|
||||||
|
waits = _job(db, s.id, queue_pos=1)
|
||||||
|
served = _job(db, o.id, queue_pos=2)
|
||||||
|
db.commit()
|
||||||
|
assert _claim_job() == served.id
|
||||||
|
assert _claim_job() is None
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(DownloadJob, waits.id).status == "queued"
|
||||||
|
db.get(User, s.id).is_suspended = False
|
||||||
|
db.commit()
|
||||||
|
assert _claim_job() == waits.id
|
||||||
|
|
||||||
|
|
||||||
# --- #3 · list cap --------------------------------------------------------------------------
|
# --- #3 · list cap --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@@ -168,7 +188,7 @@ def test_list_downloads_is_capped(db, monkeypatch):
|
|||||||
_job(db, u.id, queue_pos=i, status="done")
|
_job(db, u.id, queue_pos=i, status="done")
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
monkeypatch.setattr(dl, "_MAX_LIST", 2)
|
monkeypatch.setattr(dl, "MAX_LIST", 2)
|
||||||
rows = dl.list_downloads(user=u, db=db)
|
rows = dl.list_downloads(user=u, db=db)
|
||||||
assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk
|
assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk
|
||||||
|
|
||||||
@@ -239,7 +259,7 @@ def test_a_refusal_for_one_user_does_not_poison_the_shared_asset(db, monkeypatch
|
|||||||
that shared row — correct for an upstream failure, wrong for "YOUR storage is full", which
|
that shared row — correct for an upstream failure, wrong for "YOUR storage is full", which
|
||||||
`worker._process_job` would then hand verbatim to someone with an empty library.
|
`worker._process_job` would then hand verbatim to someone with an empty library.
|
||||||
"""
|
"""
|
||||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
over, other = User(email="over@example.com"), User(email="other@example.com")
|
over, other = User(email="over@example.com"), User(email="other@example.com")
|
||||||
db.add_all([over, other])
|
db.add_all([over, other])
|
||||||
db.flush()
|
db.flush()
|
||||||
@@ -278,3 +298,97 @@ def test_a_refusal_for_one_user_does_not_poison_the_shared_asset(db, monkeypatch
|
|||||||
assert db.get(MediaAsset, shared.id).status == "pending"
|
assert db.get(MediaAsset, shared.id).status == "pending"
|
||||||
assert db.get(MediaAsset, shared.id).error is None
|
assert db.get(MediaAsset, shared.id).error is None
|
||||||
assert db.get(DownloadJob, theirs.id).status == "queued"
|
assert db.get(DownloadJob, theirs.id).status == "queued"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_download_over_the_space_left_says_so_and_releases_the_shared_asset(
|
||||||
|
db, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
# yt-dlp refuses a file over `max_filesize` by saying so and returning with no file (pinned in
|
||||||
|
# test_formats). That is the owner's storage, not a processing fault: their refusal, in words
|
||||||
|
# true of a cap with space still left, and the shared asset goes back for the next job.
|
||||||
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
|
u = User(email="nearcap@example.com")
|
||||||
|
db.add(u)
|
||||||
|
db.flush()
|
||||||
|
db.add(
|
||||||
|
DownloadQuota(user_id=u.id, max_bytes=1000, max_concurrent=2, max_jobs=100, unlimited=False)
|
||||||
|
)
|
||||||
|
held = _sized_asset(db, 400, "near-held")
|
||||||
|
_job(db, u.id, queue_pos=0, status="done", asset_id=held.id)
|
||||||
|
shared = MediaAsset(
|
||||||
|
source_kind="youtube",
|
||||||
|
source_ref="y" * 11,
|
||||||
|
format_sig="s",
|
||||||
|
status="downloading",
|
||||||
|
ref_count=1,
|
||||||
|
)
|
||||||
|
db.add(shared)
|
||||||
|
db.flush()
|
||||||
|
mine = _job(db, u.id, queue_pos=1, status="running", asset_id=shared.id)
|
||||||
|
db.commit()
|
||||||
|
caps = []
|
||||||
|
|
||||||
|
def refuses_over_the_cap(*args, max_bytes=None, **kwargs):
|
||||||
|
caps.append(max_bytes)
|
||||||
|
formats._YDL_LOG.debug(
|
||||||
|
f"[download] File is larger than max-filesize (2000 bytes > {max_bytes} bytes). Aborting."
|
||||||
|
)
|
||||||
|
return {"id": "y" * 11, "requested_downloads": [{}]}
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline, "_extract_with_fallback", refuses_over_the_cap)
|
||||||
|
pipeline._download(mine.id, shared.id, "youtube", "y" * 11, {}, u.id)
|
||||||
|
|
||||||
|
db.expire_all()
|
||||||
|
job = db.get(DownloadJob, mine.id)
|
||||||
|
assert caps == [600]
|
||||||
|
assert (job.status, job.error_code) == ("error", "quota_exceeded")
|
||||||
|
assert job.error == dlerrors.DOWNLOAD_TOO_BIG_MESSAGE
|
||||||
|
assert db.get(MediaAsset, shared.id).status == "pending"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_download_that_outgrows_the_space_left_unannounced_is_refused_and_not_placed(
|
||||||
|
db, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
# yt-dlp's `max_filesize` only stops a plain HTTP file that announces its size: YouTube's
|
||||||
|
# fragmented formats never meet it (measured on localdev: 718 MB fetched with 1 MB left). The
|
||||||
|
# file it produced is held to what is left by its real size, as a clip is.
|
||||||
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
|
u = User(email="unannounced@example.com")
|
||||||
|
db.add(u)
|
||||||
|
db.flush()
|
||||||
|
db.add(
|
||||||
|
DownloadQuota(user_id=u.id, max_bytes=1000, max_concurrent=2, max_jobs=100, unlimited=False)
|
||||||
|
)
|
||||||
|
held = _sized_asset(db, 400, "unannounced-held")
|
||||||
|
_job(db, u.id, queue_pos=0, status="done", asset_id=held.id)
|
||||||
|
shared = MediaAsset(
|
||||||
|
source_kind="youtube",
|
||||||
|
source_ref="z" * 11,
|
||||||
|
format_sig="s",
|
||||||
|
status="downloading",
|
||||||
|
ref_count=1,
|
||||||
|
)
|
||||||
|
db.add(shared)
|
||||||
|
db.flush()
|
||||||
|
mine = _job(db, u.id, queue_pos=1, status="running", asset_id=shared.id)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
def fetched_past_the_cap(job_id, asset_id, url, spec, staging, adapter, **kw):
|
||||||
|
f = staging / "zzzzzzzzzzz.mp4"
|
||||||
|
f.write_bytes(b"v" * 2000) # 600 left
|
||||||
|
return {"id": "z" * 11, "requested_downloads": [{"filepath": str(f)}]}
|
||||||
|
|
||||||
|
monkeypatch.setattr(pipeline, "_extract_with_fallback", fetched_past_the_cap)
|
||||||
|
monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced)
|
||||||
|
pipeline._download(mine.id, shared.id, "youtube", "z" * 11, {}, u.id)
|
||||||
|
|
||||||
|
db.expire_all()
|
||||||
|
job = db.get(DownloadJob, mine.id)
|
||||||
|
assert (job.status, job.error_code, job.error) == (
|
||||||
|
"error",
|
||||||
|
"quota_exceeded",
|
||||||
|
dlerrors.DOWNLOAD_TOO_BIG_MESSAGE,
|
||||||
|
)
|
||||||
|
asset = db.get(MediaAsset, shared.id)
|
||||||
|
assert (asset.status, asset.rel_path) == ("pending", None)
|
||||||
|
assert not any(p.suffix == ".mp4" for p in tmp_path.rglob("*") if "staging" not in p.parts)
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ with "PosixPath('.') has an empty name" instead of downloading."""
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.downloads.edit import _codecs_from_streams, _concat_quote, run_ffprobe_json
|
from app.downloads.edit import _concat_quote, run_ffprobe_json
|
||||||
from app.downloads.pipeline import _media_info, _produced_file
|
from app.downloads.pipeline import _media_info, _produced_file
|
||||||
from pathlib import Path, PurePosixPath
|
from pathlib import Path, PurePosixPath
|
||||||
|
|
||||||
@@ -105,46 +105,9 @@ class TestMediaInfo:
|
|||||||
assert _media_info(info)["id"] == "vid"
|
assert _media_info(info)["id"] == "vid"
|
||||||
|
|
||||||
|
|
||||||
class TestCodecsFromStreams:
|
|
||||||
# ffprobe emits codec_name BEFORE codec_type, so the old line parser shifted every name onto
|
|
||||||
# the previous stream and an H.264+AAC mp4 came back as (aac, None) → a needless re-encode.
|
|
||||||
# _codecs_from_streams reads each stream dict by its own codec_type, immune to that ordering.
|
|
||||||
def test_reads_each_stream_by_its_own_type(self):
|
|
||||||
streams = [
|
|
||||||
{"codec_name": "h264", "codec_type": "video"},
|
|
||||||
{"codec_name": "aac", "codec_type": "audio"},
|
|
||||||
]
|
|
||||||
assert _codecs_from_streams(streams) == ("h264", "aac")
|
|
||||||
|
|
||||||
def test_audio_first_ordering_is_handled(self):
|
|
||||||
streams = [
|
|
||||||
{"codec_name": "AAC", "codec_type": "audio"},
|
|
||||||
{"codec_name": "H264", "codec_type": "video"},
|
|
||||||
]
|
|
||||||
assert _codecs_from_streams(streams) == ("h264", "aac")
|
|
||||||
|
|
||||||
def test_extra_streams_do_not_win_over_the_first_of_their_kind(self):
|
|
||||||
streams = [
|
|
||||||
{"codec_name": "h264", "codec_type": "video"},
|
|
||||||
{"codec_name": "vp9", "codec_type": "video"},
|
|
||||||
{"codec_name": "mov_text", "codec_type": "subtitle"},
|
|
||||||
{"codec_name": "opus", "codec_type": "audio"},
|
|
||||||
]
|
|
||||||
assert _codecs_from_streams(streams) == ("h264", "opus")
|
|
||||||
|
|
||||||
def test_audio_only_and_video_only(self):
|
|
||||||
assert _codecs_from_streams([{"codec_name": "opus", "codec_type": "audio"}]) == (
|
|
||||||
None,
|
|
||||||
"opus",
|
|
||||||
)
|
|
||||||
assert _codecs_from_streams([{"codec_name": "av1", "codec_type": "video"}]) == ("av1", None)
|
|
||||||
|
|
||||||
def test_empty_stream_list(self):
|
|
||||||
assert _codecs_from_streams([]) == (None, None)
|
|
||||||
|
|
||||||
|
|
||||||
class TestRunFfprobeJson:
|
class TestRunFfprobeJson:
|
||||||
"""The single ffprobe-JSON invocation point (shared by probe_codecs + probe.probe_media). A failed
|
"""The single ffprobe-JSON invocation point (`probe.probe_json`, which the stored facts and the
|
||||||
|
phone-safe rule both read). A failed
|
||||||
probe or unparseable output must degrade to None, not raise — the malformed-input safety the old
|
probe or unparseable output must degrade to None, not raise — the malformed-input safety the old
|
||||||
parse_probe_streams string parser used to own now lives here."""
|
parse_probe_streams string parser used to own now lives here."""
|
||||||
|
|
||||||
@@ -174,7 +137,7 @@ class TestRunFfprobeJson:
|
|||||||
|
|
||||||
monkeypatch.setattr(subprocess, "run", lambda *a, **k: _R())
|
monkeypatch.setattr(subprocess, "run", lambda *a, **k: _R())
|
||||||
data = run_ffprobe_json(Path("x"), entries="stream=codec_type,codec_name")
|
data = run_ffprobe_json(Path("x"), entries="stream=codec_type,codec_name")
|
||||||
assert _codecs_from_streams(data["streams"]) == ("h264", None)
|
assert data["streams"] == [{"codec_name": "h264", "codec_type": "video"}]
|
||||||
|
|
||||||
|
|
||||||
class TestConcatQuote:
|
class TestConcatQuote:
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ def test_resume_resets_a_failed_shared_asset_to_pending(db, as_user):
|
|||||||
|
|
||||||
def test_delete_releases_and_removes_the_job(db, as_user):
|
def test_delete_releases_and_removes_the_job(db, as_user):
|
||||||
u, asset, job = _seed(db, job_status="done", asset_status="ready", ref_count=1, rel_path=None)
|
u, asset, job = _seed(db, job_status="done", asset_status="ready", ref_count=1, rel_path=None)
|
||||||
r = as_user(u).request("DELETE", f"/api/downloads/{job.id}")
|
r = as_user(u).request("DELETE", f"/api/files/{job.id}")
|
||||||
assert r.status_code == 200, r.text
|
assert r.status_code == 200, r.text
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
assert db.get(DownloadJob, job.id) is None
|
assert db.get(DownloadJob, job.id) is None
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
"""An editor clip is held to its owner's storage cap by its real size, as a download and a player
|
||||||
|
clip are: its file is new storage on the owner's footprint, and nothing else checked it.
|
||||||
|
|
||||||
|
Real ffmpeg — the size the check reads is the encoder's.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.downloads import errors as dlerrors
|
||||||
|
from app.downloads import pipeline, quota, service
|
||||||
|
from app.downloads import upload as uploadmod
|
||||||
|
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_editor_clip_larger_than_what_is_left_is_refused_and_not_placed(
|
||||||
|
db, make_user, tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(settings, "files_root", str(tmp_path))
|
||||||
|
src = tmp_path / "in.mp4"
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"ffmpeg", "-y", "-loglevel", "error",
|
||||||
|
"-f", "lavfi", "-i", "testsrc2=size=160x120:rate=10:duration=2",
|
||||||
|
"-c:v", "libx264", "-pix_fmt", "yuv420p", str(src),
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
) # fmt: skip
|
||||||
|
u = make_user()
|
||||||
|
source = uploadmod.ingest_upload(
|
||||||
|
db, u.id, filename="clip.mp4", src_path=src, size=src.stat().st_size
|
||||||
|
)
|
||||||
|
job = service.enqueue_edit(db, u.id, source, {"trim": {"start_s": 0, "end_s": 1.5}})
|
||||||
|
asset = db.get(MediaAsset, job.asset_id)
|
||||||
|
job.status, asset.status = "running", "downloading" # the worker's claim
|
||||||
|
# Ten bytes left: no clip of a real video fits.
|
||||||
|
db.add(
|
||||||
|
DownloadQuota(
|
||||||
|
user_id=u.id,
|
||||||
|
max_bytes=quota.footprint(db, u.id) + 10,
|
||||||
|
max_concurrent=2,
|
||||||
|
max_jobs=100,
|
||||||
|
unlimited=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
pipeline._process_edit(job.id, asset.id)
|
||||||
|
|
||||||
|
db.expire_all()
|
||||||
|
job, asset = db.get(DownloadJob, job.id), db.get(MediaAsset, job.asset_id)
|
||||||
|
assert (job.status, job.error_code, job.error) == (
|
||||||
|
"error",
|
||||||
|
"quota_exceeded",
|
||||||
|
dlerrors.CLIP_TOO_BIG_MESSAGE,
|
||||||
|
)
|
||||||
|
assert asset.rel_path is None
|
||||||
|
assert not list(tmp_path.glob(".edits/**/*.mp4"))
|
||||||
@@ -258,7 +258,7 @@ def test_an_edit_that_crashes_is_said_once(sink, finalized, monkeypatch, tmp_pat
|
|||||||
"""The line `_process_edit`'s handler used to log before `_fail_edit` logged it again."""
|
"""The line `_process_edit`'s handler used to log before `_fail_edit` logged it again."""
|
||||||
(tmp_path / "src.mp4").write_bytes(b"x")
|
(tmp_path / "src.mp4").write_bytes(b"x")
|
||||||
monkeypatch.setattr(pipeline, "SessionLocal", _EditDb)
|
monkeypatch.setattr(pipeline, "SessionLocal", _EditDb)
|
||||||
monkeypatch.setattr(pipeline.settings, "download_root", str(tmp_path))
|
monkeypatch.setattr(pipeline.settings, "files_root", str(tmp_path))
|
||||||
monkeypatch.setattr(pipeline, "_edit_staging_dir", lambda asset_id: tmp_path / "staging")
|
monkeypatch.setattr(pipeline, "_edit_staging_dir", lambda asset_id: tmp_path / "staging")
|
||||||
|
|
||||||
def odd_spec(*a, **k):
|
def odd_spec(*a, **k):
|
||||||
@@ -300,7 +300,7 @@ def test_a_failed_download_line_does_not_put_back_what_the_adapter_took_out(sink
|
|||||||
RuntimeError("dictionary changed size during iteration"),
|
RuntimeError("dictionary changed size during iteration"),
|
||||||
RecursionError("maximum recursion depth exceeded"),
|
RecursionError("maximum recursion depth exceeded"),
|
||||||
NotImplementedError(),
|
NotImplementedError(),
|
||||||
FileNotFoundError(2, "No such file", "/downloads/Some_Channel/Season_2024/A_Private_Title"),
|
FileNotFoundError(2, "No such file", "/files/Some_Channel/Season_2024/A_Private_Title"),
|
||||||
],
|
],
|
||||||
ids=lambda e: type(e).__name__,
|
ids=lambda e: type(e).__name__,
|
||||||
)
|
)
|
||||||
@@ -499,11 +499,9 @@ def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path):
|
|||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"err",
|
"err",
|
||||||
[
|
[
|
||||||
subprocess.TimeoutExpired(
|
subprocess.TimeoutExpired(["ffmpeg", "-i", "/files/Some_Channel/A_Private_Title.mp4"], 30),
|
||||||
["ffmpeg", "-i", "/downloads/Some_Channel/A_Private_Title.mp4"], 30
|
FileNotFoundError(2, "No such file", "/files/Some_Channel/A_Private_Title.jpg"),
|
||||||
),
|
subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/files/Some_Channel/x.mp4"]),
|
||||||
FileNotFoundError(2, "No such file", "/downloads/Some_Channel/A_Private_Title.jpg"),
|
|
||||||
subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/downloads/Some_Channel/x.mp4"]),
|
|
||||||
],
|
],
|
||||||
ids=lambda e: type(e).__name__,
|
ids=lambda e: type(e).__name__,
|
||||||
)
|
)
|
||||||
@@ -523,9 +521,7 @@ def test_a_thumbnail_that_cannot_be_copied_names_no_file(sink, monkeypatch, tmp_
|
|||||||
src.write_bytes(b"\xff\xd8\xff rest")
|
src.write_bytes(b"\xff\xd8\xff rest")
|
||||||
|
|
||||||
def refuse(a, b):
|
def refuse(a, b):
|
||||||
raise PermissionError(
|
raise PermissionError(13, "Permission denied", "/files/Some_Channel/A_Private_Title.jpg")
|
||||||
13, "Permission denied", "/downloads/Some_Channel/A_Private_Title.jpg"
|
|
||||||
)
|
|
||||||
|
|
||||||
monkeypatch.setattr(editmod.shutil, "copyfile", refuse)
|
monkeypatch.setattr(editmod.shutil, "copyfile", refuse)
|
||||||
assert editmod.thumbnail_to_jpg(src, tmp_path / "out.jpg") is None
|
assert editmod.thumbnail_to_jpg(src, tmp_path / "out.jpg") is None
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
"""Every ffmpeg/ffprobe the downloads package runs over a library file names the demuxers it may use
|
||||||
|
(`edit.INPUT_GUARD`, a downloaded thumbnail's `edit.THUMB_GUARD`).
|
||||||
|
|
||||||
|
Without the list, ffmpeg picks a demuxer from the file's content and name, and both are the user's
|
||||||
|
for an upload (the remote site's for a thumbnail): a playlist-type demuxer opens whatever local paths
|
||||||
|
its entries name. These tests read the argv each builder hands to ffmpeg — the guard is a flag, so
|
||||||
|
the flag is what is held — and scan the package for an ffmpeg/ffprobe call written without one.
|
||||||
|
|
||||||
|
Not in scope: `app/plex`, whose ffmpeg reads the admin's own Plex library, not user-supplied bytes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.downloads import edit, playable
|
||||||
|
|
||||||
|
DOWNLOADS = Path(__file__).resolve().parents[1] / "app" / "downloads"
|
||||||
|
# The demuxers that read paths named inside a file. `concat` is allowed only where the list is ours.
|
||||||
|
PLAYLIST_DEMUXERS = {"hls", "dash", "concat", "image2", "ffconcat", "tee"}
|
||||||
|
MEDIA = set(edit.MEDIA_DEMUXERS)
|
||||||
|
|
||||||
|
|
||||||
|
def _whitelists(cmd: list[str]) -> list[set[str] | None]:
|
||||||
|
"""For each `-i` in `cmd`, the demuxers the `-format_whitelist` given since the previous input
|
||||||
|
allows — None when that input has none."""
|
||||||
|
out, current = [], None
|
||||||
|
for i, tok in enumerate(cmd):
|
||||||
|
if tok == "-format_whitelist":
|
||||||
|
current = set(cmd[i + 1].split(","))
|
||||||
|
elif tok == "-i":
|
||||||
|
out.append(current)
|
||||||
|
current = None
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_media_guarded(cmd: list[str], extra: set[str] = frozenset()) -> None:
|
||||||
|
lists = _whitelists(cmd)
|
||||||
|
assert lists, cmd # a builder that opens no input is not what this test is about
|
||||||
|
for allowed in lists:
|
||||||
|
assert allowed is not None, f"an input with no -format_whitelist: {cmd}"
|
||||||
|
assert allowed <= MEDIA | extra, allowed - MEDIA - extra
|
||||||
|
assert not (allowed - extra) & PLAYLIST_DEMUXERS
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_media_list_names_no_demuxer_that_reads_other_paths():
|
||||||
|
assert not MEDIA & PLAYLIST_DEMUXERS
|
||||||
|
assert edit.INPUT_GUARD == ["-format_whitelist", ",".join(edit.MEDIA_DEMUXERS)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_phone_safe_conversion_is_guarded():
|
||||||
|
p = playable.Plan("mp4", True, True, True, True)
|
||||||
|
_assert_media_guarded(playable.build_cmd(Path("in.mov"), Path("out.mp4"), p))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"spec",
|
||||||
|
[
|
||||||
|
{"trim": {"start_s": 1, "end_s": 3}, "accurate": False},
|
||||||
|
{"trim": {"start_s": 1, "end_s": 3}, "accurate": True},
|
||||||
|
{"crop": {"x": 0, "y": 0, "w": 101, "h": 51}},
|
||||||
|
],
|
||||||
|
ids=["fast trim", "accurate trim", "crop"],
|
||||||
|
)
|
||||||
|
def test_a_clip_is_guarded(spec):
|
||||||
|
spec = edit.normalize_edit_spec(spec)
|
||||||
|
_assert_media_guarded(edit.build_edit_cmd(Path("in.mp4"), Path("out.mp4"), spec, "mp4"))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("accurate", [True, False], ids=["accurate join", "fast join"])
|
||||||
|
def test_a_join_is_guarded(accurate, tmp_path):
|
||||||
|
segs = [{"start_s": 0, "end_s": 1}, {"start_s": 2, "end_s": 3}]
|
||||||
|
spec = edit.normalize_edit_spec({"segments": segs, "accurate": accurate})
|
||||||
|
cmd, _ = edit.build_concat_plan(Path("in.mp4"), Path("out.mp4"), spec, "mp4", tmp_path)
|
||||||
|
# The fast join reads a concat list WE write, so the list itself may be opened as concat.
|
||||||
|
_assert_media_guarded(cmd, extra=set() if accurate else {"concat"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_fast_join_list_gives_each_entry_the_media_list_without_concat(tmp_path):
|
||||||
|
# ffmpeg hands the list's whitelist (with `concat`) down to the library file each entry names;
|
||||||
|
# the entry's own `option format_whitelist` must replace it, or that file may open as a list.
|
||||||
|
segs = [{"start_s": 0, "end_s": 1}, {"start_s": 2, "end_s": 3}]
|
||||||
|
spec = edit.normalize_edit_spec({"segments": segs, "accurate": False})
|
||||||
|
_, prep = edit.build_concat_plan(Path("in.mp4"), Path("out.mp4"), spec, "mp4", tmp_path)
|
||||||
|
(text,) = prep.values()
|
||||||
|
lines = text.splitlines()
|
||||||
|
entries = [i for i, ln in enumerate(lines) if ln.startswith("file ")]
|
||||||
|
assert len(entries) == 2
|
||||||
|
for i in entries:
|
||||||
|
key, _, allowed = lines[i + 1].partition(" format_whitelist ")
|
||||||
|
assert key == "option", lines[i + 1]
|
||||||
|
assert set(allowed.split(",")) == MEDIA
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||||
|
def test_the_fast_join_still_joins_a_real_file(tmp_path):
|
||||||
|
src = tmp_path / "in.mp4"
|
||||||
|
subprocess.run(
|
||||||
|
["ffmpeg", "-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc=size=64x48:rate=10",
|
||||||
|
"-f", "lavfi", "-i", "sine=frequency=440", "-t", "4", "-c:v", "libx264",
|
||||||
|
"-pix_fmt", "yuv420p", "-c:a", "aac", str(src)],
|
||||||
|
check=True,
|
||||||
|
) # fmt: skip
|
||||||
|
segs = [{"start_s": 0, "end_s": 1}, {"start_s": 2, "end_s": 3}]
|
||||||
|
spec = edit.normalize_edit_spec({"segments": segs, "accurate": False})
|
||||||
|
out = tmp_path / "out.mp4"
|
||||||
|
cmd, prep = edit.build_concat_plan(src, out, spec, "mp4", tmp_path)
|
||||||
|
for p, content in prep.items():
|
||||||
|
p.write_text(content, encoding="utf-8")
|
||||||
|
subprocess.run(cmd, check=True, capture_output=True)
|
||||||
|
assert (edit.probe_duration(out) or 0) > 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_poster_and_the_filmstrip_are_guarded():
|
||||||
|
_assert_media_guarded(edit.build_poster_cmd(Path("in.mp4"), Path("p.jpg"), 2.0))
|
||||||
|
geom = edit.storyboard_geometry(60)
|
||||||
|
_assert_media_guarded(edit.build_storyboard_cmd(Path("in.mp4"), Path("s.jpg"), 60, geom))
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_downloaded_thumbnail_opens_as_a_single_image_only():
|
||||||
|
(allowed,) = _whitelists(edit.build_thumbnail_jpg_cmd(Path("t.webp"), Path("t.jpg")))
|
||||||
|
assert allowed == {"png_pipe", "jpeg_pipe", "webp_pipe"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_ffprobe_is_guarded(monkeypatch):
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
class _Done:
|
||||||
|
stdout = "{}"
|
||||||
|
|
||||||
|
def _run(args, **_k):
|
||||||
|
seen.append(args)
|
||||||
|
return _Done()
|
||||||
|
|
||||||
|
monkeypatch.setattr(edit.subprocess, "run", _run)
|
||||||
|
edit.run_ffprobe_json(Path("x.mp4"), show_streams=True)
|
||||||
|
edit.probe_duration(Path("x.mp4"))
|
||||||
|
assert len(seen) == 2
|
||||||
|
for args in seen:
|
||||||
|
i = args.index("-format_whitelist")
|
||||||
|
assert set(args[i + 1].split(",")) == MEDIA
|
||||||
|
|
||||||
|
|
||||||
|
def _unguarded_calls(source: str) -> list[int]:
|
||||||
|
"""Lines of a list literal that starts an ffmpeg/ffprobe command or hands one an input (`-i`)
|
||||||
|
and carries no `*…GUARD` / `*guard` splat and no "-format_whitelist" of its own. A builder that
|
||||||
|
adds its input in a later `+=` must guard that list — that is where the input is."""
|
||||||
|
bad = []
|
||||||
|
for node in ast.walk(ast.parse(source)):
|
||||||
|
if not isinstance(node, ast.List):
|
||||||
|
continue
|
||||||
|
consts = {e.value for e in node.elts if isinstance(e, ast.Constant)}
|
||||||
|
if not consts & {"-i", "ffprobe"}:
|
||||||
|
continue
|
||||||
|
guarded = "-format_whitelist" in consts or any(
|
||||||
|
isinstance(e, ast.Starred)
|
||||||
|
and isinstance(v := e.value, ast.Name | ast.Attribute)
|
||||||
|
and (v.id if isinstance(v, ast.Name) else v.attr).upper().endswith("GUARD")
|
||||||
|
for e in node.elts
|
||||||
|
)
|
||||||
|
if not guarded:
|
||||||
|
bad.append(node.lineno)
|
||||||
|
return bad
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_ffmpeg_call_in_the_package_opens_an_input_unguarded():
|
||||||
|
found = {
|
||||||
|
f"{p.name}:{line}"
|
||||||
|
for p in sorted(DOWNLOADS.rglob("*.py"))
|
||||||
|
for line in _unguarded_calls(p.read_text(encoding="utf-8"))
|
||||||
|
}
|
||||||
|
assert found == set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_scan_sees_an_unguarded_call():
|
||||||
|
# The scan above is green over nothing if it cannot see one: both shapes it must catch.
|
||||||
|
assert _unguarded_calls('cmd = ["ffmpeg", "-y", "-i", src]') == [1]
|
||||||
|
assert _unguarded_calls('a = ["ffprobe", "-v", "error", str(p)]') == [1]
|
||||||
|
assert _unguarded_calls('cmd = [*INPUT_GUARD, "-i", src]') == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||||
|
@pytest.mark.parametrize("ext", ["png", "webp", "jpg"])
|
||||||
|
def test_the_thumbnail_guard_still_opens_every_thumbnail_a_download_keeps(ext, tmp_path):
|
||||||
|
# `pipeline._THUMB_SUFFIXES`: what the guard must still let through (a .jpg that is really a
|
||||||
|
# PNG goes the ffmpeg route too — `thumbnail_to_jpg` copies only true JPEG bytes).
|
||||||
|
src = tmp_path / f"t.{ext}"
|
||||||
|
fmt = ["-f", "image2", "-c:v", "png"] if ext == "jpg" else []
|
||||||
|
subprocess.run(
|
||||||
|
["ffmpeg", "-y", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=size=64x48",
|
||||||
|
"-frames:v", "1", *fmt, str(src)],
|
||||||
|
check=True,
|
||||||
|
) # fmt: skip
|
||||||
|
out = edit.thumbnail_to_jpg(src, tmp_path / "poster.jpg")
|
||||||
|
assert out is not None and edit._is_jpeg(out)
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user