Compare commits
142
Commits
v0.82.0
...
8c10e57208
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8c10e57208 | ||
|
|
dd040adc96 | ||
|
|
4ab128ccab | ||
|
|
a28193a00c | ||
|
|
986bc8d01e | ||
|
|
07e3ea3737 | ||
|
|
b2eaa9e00b | ||
|
|
791ba5f507 | ||
|
|
7789db8240 | ||
|
|
f92cd9017f | ||
|
|
4bfba4aa86 | ||
|
|
86bc008a81 | ||
|
|
fac5c4465a | ||
|
|
9c9b1cf6d5 | ||
|
|
c3d88d9b20 | ||
|
|
569190480d | ||
|
|
5883ea509d | ||
|
|
a210992bfc | ||
|
|
0517de1f59 | ||
|
|
b59fd38a9c | ||
|
|
88a166e4aa | ||
|
|
07766c43e8 | ||
|
|
bb3c6dec45 | ||
|
|
3cf8f04d73 | ||
|
|
2b56c5c2eb | ||
|
|
c06a456a9a | ||
|
|
864f673ef5 | ||
|
|
6c63df6704 | ||
|
|
a5bcd9fca2 | ||
|
|
618ef812e6 | ||
|
|
e299c66121 | ||
|
|
11aef4ab7a | ||
|
|
4e52bdd4ca | ||
|
|
595490cb98 | ||
|
|
7537882235 | ||
|
|
47d968eeea | ||
|
|
e7f38396a2 | ||
|
|
f569b19c49 | ||
|
|
e795d191c5 | ||
|
|
4ad7183102 | ||
|
|
2c3aaec62d | ||
|
|
70598d47a3 | ||
|
|
5858e7b513 | ||
|
|
5346efcd10 | ||
|
|
6169e32ca5 | ||
|
|
90523284ab | ||
|
|
17bfea8754 | ||
|
|
2a9ddb78ac | ||
|
|
e887233210 | ||
|
|
b17d92d7c2 | ||
|
|
90ee34a1b4 | ||
|
|
118fc70e08 | ||
|
|
636372511b | ||
|
|
1354f5b503 | ||
|
|
98ec1ffcab | ||
|
|
18c11ca437 | ||
|
|
ca5e446fbd | ||
|
|
4c7ab12a3e | ||
|
|
c6b552e75f | ||
|
|
3cd0c525e2 | ||
|
|
f130a1c05a | ||
|
|
bbd31c7143 | ||
|
|
8731508645 | ||
|
|
b5d37ca142 | ||
|
|
eb0507f942 | ||
|
|
6d09b94bbb | ||
|
|
5274216981 | ||
|
|
75d89613d0 | ||
|
|
8fb276aa90 | ||
|
|
78a9324381 | ||
|
|
8a50538275 | ||
|
|
b4aea7fa5b | ||
|
|
92c305bab9 | ||
|
|
32fe0768b6 | ||
|
|
415a6c16f9 | ||
|
|
8c51062e5b | ||
|
|
63bf1bca78 | ||
|
|
c7a377396b | ||
|
|
8036251589 | ||
|
|
71d7646bf1 | ||
|
|
3eb0f41962 | ||
|
|
a0114deff1 | ||
|
|
d89d887a9b | ||
|
|
f8a31d48cf | ||
|
|
7f17b17c13 | ||
|
|
1f81684b6e | ||
|
|
33fddacee2 | ||
|
|
12121255dd | ||
|
|
4a7b65d187 | ||
|
|
47af4d5c05 | ||
|
|
8a83bc0e9f | ||
|
|
41b652f8fd | ||
|
|
545996f807 | ||
|
|
267fd7f70f | ||
|
|
6d49da17fe | ||
|
|
6a92585098 | ||
|
|
31dc4184f5 | ||
|
|
86971c574d | ||
|
|
785080f3da | ||
|
|
76b5f7c93e | ||
|
|
1cfe6a7f59 | ||
|
|
33fe895539 | ||
|
|
5c5cabbe5a | ||
|
|
4f37c41021 | ||
|
|
f7faa19ee2 | ||
|
|
0b4b268678 | ||
|
|
d349bc9084 | ||
|
|
dcd8a9566a | ||
|
|
6de2cd950d | ||
|
|
645d14f45d | ||
|
|
7360339039 | ||
|
|
473d9f02f2 | ||
|
|
d2e225666b | ||
|
|
efd95fca01 | ||
|
|
060deb89d2 | ||
|
|
4869d6c108 | ||
|
|
d20317e0f1 | ||
|
|
d2ba65e383 | ||
|
|
cf7ec5e9ec | ||
|
|
f76a2afbf1 | ||
|
|
889d2acf52 | ||
|
|
782362b808 | ||
|
|
5a42d76e8a | ||
|
|
ed4ab4e13c | ||
|
|
b8bff504d8 | ||
|
|
1630a3df3c | ||
|
|
14b5fe2e31 | ||
|
|
d260aabfd7 | ||
|
|
d832bb19ce | ||
|
|
59efc8f2e8 | ||
|
|
e4ffd59ca7 | ||
|
|
3709b71a94 | ||
|
|
cedebe0ffb | ||
|
|
fdc1eabbe0 | ||
|
|
84f4b20e5a | ||
|
|
828dad015c | ||
|
|
82914b56ca | ||
|
|
ccd5f3739a | ||
|
|
f3c0b7be23 | ||
|
|
53b50231ff | ||
|
|
5703c4d713 | ||
|
|
013f19faca |
@@ -28,6 +28,14 @@ ALLOWED_EMAILS=
|
||||
# Admin emails (subset of the above) get the admin role.
|
||||
ADMIN_EMAILS=
|
||||
|
||||
# Logging (E22). LOG_LEVEL is the BOOTSTRAP level — the level a process starts at and the only one
|
||||
# the worker/download child read; the API's running level is the admin Configuration page's
|
||||
# `log_level` knob, which takes effect without a restart. TRACE | DEBUG | INFO | WARNING | ERROR, in
|
||||
# any letter case; any other value stops the boot with an error. LOG_FORMAT is "json" (one object per
|
||||
# line — what a log collector ingests) or "text" (a readable line for a local terminal).
|
||||
LOG_LEVEL=INFO
|
||||
LOG_FORMAT=json
|
||||
|
||||
# Optional: origin of a separately-served frontend dev server (enables CORS). Leave empty in production.
|
||||
FRONTEND_ORIGIN=
|
||||
|
||||
|
||||
+172
@@ -7,6 +7,178 @@ edit the release notes and re-run the generator.
|
||||
|
||||
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
||||
|
||||
## 0.88.0 — 2026-09-18
|
||||
|
||||
When something goes wrong, you can now turn on detailed logging for yourself for 24 hours — and read your own log on the spot, before any of it is sent anywhere. Siftlode also has a privacy policy and terms at last, in both languages, saying exactly what is recorded and for how long.
|
||||
|
||||
### Features
|
||||
|
||||
- Settings → Diagnostics: choosing Detailed or Full changes nothing until you confirm a box that tells you what will be recorded, that the server keeps it, for how long, and that it switches itself off after 24 hours. Going back to Basic needs no confirmation and takes effect at once.
|
||||
- “My log”, on the same tab, shows what your browser has recorded — newest first, any line openable. Nothing leaves the browser before you have consented.
|
||||
- What you search for and filter by is recorded only at Full, and only under your own consent — never an administrator's. Passwords, tokens, cookies and email addresses are recorded at no level, by anyone.
|
||||
- Your last sign-in is shown in your account panel, and in the administrator's user list.
|
||||
- A privacy policy and terms of service, in Hungarian and English, covering what the server logs and what each diagnostics level adds.
|
||||
- Administrators get a Diagnostics section in Configuration: the log level takes effect without a restart, and stored logs are cleaned up on a schedule (7 days by default).
|
||||
- Every setting with a fixed set of values is now picked from those values instead of typed in, so a wrong one can no longer be saved.
|
||||
|
||||
### Fixes
|
||||
|
||||
- Every message arrived twice if you left a conversation and came back quickly: two chat connections stayed open where there should have been one.
|
||||
- Settings was unusable on a phone — the tab rail took the width and left almost nothing for the panel beside it.
|
||||
- Dates and numbers follow the language you picked in Siftlode, not the one your computer happens to be set to. An English Siftlode on a Hungarian machine no longer shows Hungarian dates.
|
||||
- An administrator setting shown as a dropdown opened on its first option instead of the one actually in force.
|
||||
- In Hungarian, the audit log and the new log settings had names you could not tell apart.
|
||||
- The scheduled feed poll now covers only channels somebody actually follows — on this instance 582 of 979 polled channels had no follower at all. The trade-off is deliberate: new uploads from a channel nobody follows no longer appear in the shared Library.
|
||||
|
||||
### Internal
|
||||
|
||||
- One logging mechanism replaced four. The API, the worker, the downloader and the migration tool now write the same structured line; secrets are scrubbed where the line is written rather than at each of ~125 places that write one; and every request carries an id that joins a line from your browser to the server line of the same request. The browser's log uses the same shape and the same redaction rules, checked from both sides against one shared set of cases.
|
||||
- Two adversarial review rounds over the epic: 56 findings raised, 32 survived three independent attempts to refute each, all fixed. Every regression test was proven to fail with its own fix removed — which is how a test that passed only on leftover state from an earlier test was caught, and how a focus bug that jsdom could not reproduce at all was settled in a real browser.
|
||||
|
||||
## 0.87.1 — 2026-09-16
|
||||
|
||||
A YouTube video that YouTube will only show to a signed-in viewer can now be downloaded: paste your YouTube cookies in Settings the way the failure message already told you to — until now that list had no YouTube in it.
|
||||
|
||||
### Features
|
||||
|
||||
- YouTube joins the sites you can store download cookies for (Settings → Account → Download authentication). A signed-in YouTube download automatically uses the players that accept your login.
|
||||
|
||||
### Fixes
|
||||
|
||||
- A YouTube download that stops at “Sign in to confirm you’re not a bot” says so in your language and points at a setting that now exists. If you had already added cookies, it tells you they were refused instead of asking for them again.
|
||||
- A video YouTube describes but then refuses to hand over is tried again with another player instead of failing at once, and it is no longer called a network error — nothing was wrong with the network.
|
||||
|
||||
### Internal
|
||||
|
||||
- The gated video was measured end to end on an isolated stack before and after the fix: every player client, two yt-dlp versions, and finally the real download (519 MB) with cookies attached. One deep-review round — 25 findings raised, none survived adversarial verification, and the one real defect came out of the verification itself: a comment of mine named an error yt-dlp does not raise there. Running it settled what reading it could not.
|
||||
|
||||
## 0.87.0 — 2026-09-15
|
||||
|
||||
A shared or library video keeps playing when you switch apps on an iPad, and a new tab opens as the account it belongs to without flashing another page first. Facebook reels keep their titles and an mp3 stays an mp3.
|
||||
|
||||
### Features
|
||||
|
||||
- Background play on iPad (Brave): the shared-video page, the library player and the Plex player keep playing in another app and on the locked screen.
|
||||
- The shared-video page has a Repeat toggle, on by default.
|
||||
- Switching modules no longer flashes a loading screen: the pages load quietly in the background, and a slow answer shows its spinner only after a moment.
|
||||
|
||||
### Fixes
|
||||
|
||||
- A new tab honours your Startup choice and opens as its own account — no Home page flash, no other account's language or settings on the first frame.
|
||||
- A signed-in Facebook reel keeps its real title instead of “Video” or “Facebook”, and an mp3 download stays an mp3 (it used to be wrapped into an mp4 an iPad would not play).
|
||||
- A tab whose account was signed out, suspended or signed out everywhere acts as nobody and comes back as the account still signed in, instead of writing into another account or sticking on the sign-in page.
|
||||
- Switching or adding an account no longer saves the video you were watching into the other account's resume position.
|
||||
- A new tab keeps the account's notification history instead of replacing it with its own first notice.
|
||||
- The shared-video page tells a failed load, a wrong password, too many attempts and an unreachable server apart, with Retry where it helps. Its password limit counts per viewer, so one person guessing cannot lock everyone else out.
|
||||
- Form errors on sign-in, the message passphrase, password change and setup are read out by screen readers — again on a second identical failure — and the setup wizard names what is wrong instead of a generic error.
|
||||
|
||||
### Internal
|
||||
|
||||
- WebKit and iPad projects in the Playwright suite; the Vite dev proxy forwards websockets.
|
||||
- Four deep-review rounds (12 → 8 → 8 → 8, the last one targeted), every regression test mutation-checked; half of the later rounds' findings were older defects, fixed here.
|
||||
|
||||
## 0.86.0 — 2026-09-14
|
||||
|
||||
A failed download now tells you why it failed and what you can do about it, instead of repeating yt-dlp's raw error. Facebook comes first: downloads that used to die on “Cannot parse data” go through, and a Facebook group post downloads its own video instead of a different one from the page.
|
||||
|
||||
### Features
|
||||
|
||||
- A failure says why, in your language: the row names the cause (“Facebook needs you to be signed in”, “Facebook did not accept your cookies”, “No video found in this post”) and its details window says what to do next. The raw error text is still there below it.
|
||||
- A Facebook download with your cookies that stops at “Cannot parse data” is tried again as Chrome, which gets past it.
|
||||
- A Facebook group post link downloads the post's own video. Before, yt-dlp either found nothing or quietly downloaded another video from the page; a post without a video now says so.
|
||||
- Retry and Re-download use a cookie you added after the failure, so the Retry the message asks for actually goes out signed in.
|
||||
|
||||
### Fixes
|
||||
|
||||
- A Re-download of a download that needed your cookies used to rebuild without them.
|
||||
- A job queued behind a download that already failed shows that failure's explanation instead of a bare “Download failed”.
|
||||
- A download yt-dlp reports no length for takes its length from the file, so its Edit / trim action appears.
|
||||
- Pressing Pause, Stop or Remove twice, or in two tabs, can no longer free a file another download still uses.
|
||||
- The confirm buttons in Downloads name the action (Stop download, Re-download, Remove) instead of repeating the question.
|
||||
|
||||
### Internal
|
||||
|
||||
- Four sprints, each Facebook behaviour measured on the development server with a real cookie before it was built. Two epic-end review rounds (20 → 17 fixed, then a targeted 4 → 4), every regression test mutation-checked.
|
||||
|
||||
## 0.85.0 — 2026-09-13
|
||||
|
||||
Siftlode can now be driven from the keyboard, and Back, dialogs and menus stop losing your place. j/k and the arrow keys walk every list and grid, Enter opens what you are on, and Back puts you on the item you opened a view from instead of at the top of the page.
|
||||
|
||||
### Features
|
||||
|
||||
- j/k and the arrow keys walk the feed, the library, a channel page, the Home rails, every Plex grid and rail, the Channels, Playlists and Downloads tables and the conversation list; Enter opens the item you are on. After you hide or delete an item, the next j goes on from where it was.
|
||||
- Back returns you to the item a view was opened from — the Plex title you opened from a Home rail, the channel you opened from its row — with the keyboard still on it.
|
||||
- Shortcuts to get around: s jumps to the search field, g-chords jump to a module, and the cheat sheet (h) names every key in the language you are reading. PageUp/PageDown move a page at a time in the command palette and in every dropdown list.
|
||||
- The command palette ranks what you typed first: “theme” now puts the Theme rows above a channel whose handle happens to contain it.
|
||||
- A card reached with Tab or j shows one ring around the whole card, and a screen reader calls it by its title.
|
||||
|
||||
### Fixes
|
||||
|
||||
- Back and overlays keep their books straight: Back after navigating over About also closes About, a late clean-up no longer undoes the navigation you just made, and Shift+W on Home closes the player.
|
||||
- Dialogs and menus hand the keyboard back where it came from. Escape from a card’s context menu returns to the card, not its thumbnail link; closing the player under another dialog no longer leaves the page unscrollable; a button that is busy with a request keeps the focus instead of dropping it to the page.
|
||||
- A write that fails says so, with the reason — several saves, share links and admin actions used to fail silently or in English under the Hungarian interface.
|
||||
- A split video export that fails part-way (the job limit) tells you how many clips were added and leaves only the rest selected, so trying again does not queue the first ones twice.
|
||||
- “Feed” is called Feed everywhere, in both languages.
|
||||
|
||||
### Internal
|
||||
|
||||
- Five sprints: history integrity, one dialog/focus contract, palette ranking and accessibility, the keyboard layer, and a Hungarian/English keyboard-only pass. Focus and event-order questions were measured in a real browser before each fix, not argued.
|
||||
- Three epic-end review rounds — 19 → 14, 18 → 12 and a targeted 14 → 14 — each regression test mutation-checked. The last round's findings all sat inside the previous round's own fixes, which is where the loop was closed.
|
||||
|
||||
## 0.84.0 — 2026-09-12
|
||||
|
||||
A bug-fix release, from a list you wrote while using Siftlode. Two of these were destroying your own data: a YouTube video rewound itself when you reopened it, and clearing a resume point on a half-watched Plex item told Plex nothing. Subtitles now appear in seconds instead of minutes, the Plex player's bitrate is a real measurement rather than a guess, and a channel you turn full history on for fills in front of you instead of looking frozen.
|
||||
|
||||
### Features
|
||||
|
||||
- Subtitles arrive in seconds. Picking an embedded subtitle track used to take about two and a half minutes on a large film, and switching from Hungarian to English cost the same again — because the cost was never the conversion, it was reading the whole file. It now reads a window around where you are, shows that first, and finishes the rest behind it; every text track rides the SAME read, so switching tracks is instant. On a 17 GB film: 154 s before, 8 s for the window now. Nothing blocks while it works, there is a “Preparing subtitles” pill instead of a frozen player, and a track that can never be shown finally says so.
|
||||
- The Plex player's info panel tells you the truth. Bitrate is measured from the bytes actually arriving, not copied off the manifest — and it shows a dash until there is something to measure rather than a confident wrong number. It also lists the audio and the subtitle track you are on, which it never did.
|
||||
- A new admin switch: loudness normalisation for Plex playback (Configuration → Plex). Off by default, because it makes a film markedly louder and narrows its dynamic range — that is a taste call, not a fix. On a 5.1 film it moved the measured loudness from −35.2 to −17.2 LUFS, and it does not touch A/V sync.
|
||||
- Collections can be sorted. The collection page had no sort control at all and always ordered by release year — it now carries the same Sort control as the library grid, with the same options, and your choice survives a reload without re-sorting the library behind it. A collection opened from a shared link also shows its real name instead of a raw id.
|
||||
- Remove a video from a playlist without leaving the player, and the player steps to whatever followed instead of closing. Above the move-to submenu there is now a one-click “move to the list you last moved to”.
|
||||
- Live YouTube search results can be sorted — by views, date or length, over the results already on screen, so sorting rearranges what you are looking at instead of running a different search.
|
||||
- The channel name in the player is a real link again: middle-click and Ctrl-click open it in a new tab, the way the YouTube icon beside it always did, while a plain click still takes the in-app route.
|
||||
|
||||
### Fixes
|
||||
|
||||
- A YouTube video no longer rewinds when you reopen it. Closing at 13:50 and coming back put you at 12:31 — the checkpoint reached the server but not the page's own cache, so the stale position won. It now goes forward: 811 → 816 → 825 s across reopens.
|
||||
- “Clear resume point” works on a half-watched Plex item. The push to Plex was skipped for exactly the case you would use it on, so Plex kept the old offset and handed it straight back on the next import. Verified against the real server, including a full 44,885-item re-import.
|
||||
- Turning full history on for a channel fills the page in front of you. The grid and the item count stayed frozen for the best part of a minute while the header counted up, because the request that starts the backfill returns before anything is stored. Measured: the count now moves 54 → 139 within about twelve seconds, no reload.
|
||||
- Starting or seeking a Plex film is about eight seconds faster. Every session start and every seek waited out a dead eight-second timer for a file the transcoder never writes — on a film with more than one audio track, which is most of them. A multi-audio session now starts in 0.9 s cold and 0.7 s on a seek.
|
||||
- A Plex session the server no longer has now recovers or says so, instead of spinning forever. A stream that kept being lost could restart itself indefinitely behind a spinner that never became an error.
|
||||
- A Plex stream request for something that cannot exist answers immediately rather than holding the server for 20–30 seconds. Enough of those at once could stall the whole app.
|
||||
- The Plex player no longer counts one viewing twice, and a failed “watched” mark no longer half-happens: it used to leave the item looking watched in the app while the server disagreed, or quietly mark it watched anyway a few seconds after telling you it had failed.
|
||||
- Dragging a Plex playlist into a new order sticks. Leaving and coming straight back showed the old order, and the next drag saved that.
|
||||
- A refused action says WHY, in the language you are reading. Several places showed a generic “couldn't do that” — or nothing at all — where the server had explained itself, including the Plex “Add to playlist” dialog, the collection editor and the upload dialog. The demo account gets its reason back too.
|
||||
- Dialogs behave for the keyboard. Opening one puts the cursor where you would expect (the jump-to box opens on its search field, not its close button), closing one hands the focus back to what you opened it from, and a dialog that closes underneath another one no longer steals the focus out of the one still on screen.
|
||||
- The regular micro-stutter in the Plex player is NOT Siftlode's, and this release does not pretend to fix it. Measured per frame: the player delivers every source frame with nothing dropped; the hold comes from a 23.976 fps film on a 120.0148 Hz display, and it reproduces identically outside Siftlode. Setting the display to 119.88 Hz removes it for every application, not just this one.
|
||||
|
||||
### Internal
|
||||
|
||||
- The round started from sixteen items written from real use, not from an audit. Eleven were fixed, one was measured and closed without a code change (the display cadence above), two are parked for a reproducible case, and one — playlist multi-select and bulk actions — was scoped out to its own sprint. The measurements are the point: item 8's “35 s” turned out to be one file's number, and the real law is bytes-ffmpeg-must-read divided by ~140 MB/s, which is why the fix is a window rather than a faster conversion.
|
||||
- Seven deep-review rounds followed, each reviewing the previous round's own fix commits: 32 → 24 → 43 → 31 → 22 → 26 → 18 raised. Round 7 was run TARGETED rather than full — four reviewers on the six surfaces round 6 had changed, and adversarial verification spent only where a wrong fix could re-break the previous round's — which is seven agents instead of about thirty-six for two HIGH findings. Every regression test added in a fix round was mutation-checked: reverted, run, and confirmed red for the defect it is named after. Thirteen of round 7's mutations were, and two were instructive — one guard no test could drive, and one round-6 test whose mock modelled a server that accepts a write and then serves the pre-write state.
|
||||
- What the eight-lane gate structurally cannot catch, and what this round taught about it: every real defect of the last three rounds was a wrong belief about an external system, not broken logic. ffmpeg substitutes the VARIANT before the sequence in its segment filenames, so a guard written the other way round was inert on every multi-audio file and a warm-up waited eight seconds for a name that is never written. A reclaim budget's window was anchored at the first failure rather than the last, so any failure cadence between one and two minutes reset it forever — invisible to two tests that spaced their cases at zero and ten minutes. jsdom performs no default Tab, so a focus test can be green with no focus trap installed at all. The conclusion carried forward is to measure the boundary when writing the code, not when reviewing it: one ffmpeg run costs pennies, a review round costs a session.
|
||||
|
||||
## 0.83.0 — 2026-09-09
|
||||
|
||||
Siftlode now opens on a page that answers one question: what should I watch right now. You say how much time you have — ten minutes, half an hour, a whole evening — and it gives you one suggestion, from YouTube or from Plex, with a button that picks up where you left off. Under it sits a shelf of the things you always come back to, which you fill yourself.
|
||||
|
||||
### Features
|
||||
|
||||
- A new Home page, and it is where Siftlode starts. It asks how much time you have — 10 minutes, 30 minutes, an evening, or it does not matter — and answers with ONE suggestion rather than a wall of thumbnails. The question is on top, the answer directly under it. If you would rather land on the feed as before, Settings → Appearance → Startup turns it off.
|
||||
- The time you have picks the source, not just the length. Ten minutes gets you a short video; an evening is allowed to offer a film. It matches on the time you have LEFT in something, so a series episode you are four minutes from finishing counts as a four-minute answer.
|
||||
- The suggestion is one of three things, and it says which: something you left half-finished, something off your comfort shelf, or a surprise. Continue takes you back to the second you stopped. Something else rerolls it — there is no separate surprise-me page, the reroll is it.
|
||||
- A comfort shelf you fill yourself, for the media you put on without thinking. It takes a whole channel, a whole show, a film, a single video or a playlist — because comfort viewing is put on TheVR, not play this one episode. One click resolves it: a show gives you the next unwatched episode, a channel its newest unwatched video. The shelf can be reordered into the order you want.
|
||||
- Pin from wherever you are: the same ⋯ menu — or a right-click — on a feed video, a channel row, a playlist, a Plex poster or a Plex row now carries a Comfort shelf entry. Feed video cards answer a right-click for the first time.
|
||||
- Continue watching, on both sides at once. The rail merges what you left half-watched on YouTube with what you left half-watched on Plex, most recently touched first — the YouTube half of that did not exist before.
|
||||
- Shift+W jumps to Home from anywhere, including out of a player, and it is in the keyboard sheet. The jump-to box (Ctrl+K) gained a Surprise me row, and typing what should I watch finds Home itself.
|
||||
|
||||
### Internal
|
||||
|
||||
- E24, four sprints. S1 built a discover read-model as its own backend package — the unified card shape maps the EXISTING Plex and feed serializers rather than adding a base serializer, the rails reuse the feed query layer and the Plex rollup, and the tiered hero rule is one shared function that returns the tier it chose, so the UI prints an honest kicker instead of guessing. The shelf is a typed comfort_pins table with a one-of check constraint and a real foreign key per kind, not a polymorphic string reference: a deleted Plex item must not leave a dead pin on the shelf. S2 built the page on the existing carousel and the two existing card components rather than a third. S3 wired pinning through the context-menu input path E10 shipped, which absorbed that epic's parked video-card item. S4 added the ways in.
|
||||
- The closing review ran four rounds, 46 → 23 → 19 → 15 → 14, each round reviewing the previous round's own fix diff, with three adversarial verifiers per finding in rounds one to three. Four of round one's findings died at 2-of-3 and are recorded so they are not raised again. What the eight-lane gate could not have caught: the resume rail built its own query and dropped the per-user channel blocklist, so a video from a channel you had just banned could become the landing page's suggestion; the shelf numbered a new pin by row COUNT while nothing renumbers, so after any non-tail unpin a pin landed mid-shelf and could collide; the Home tile and the show page disagreed about which episode comes next, one taking the first in-progress episode and the other the last; and Shift+W was dead whenever anything was open, because the page change unmounted the player and its cleanup reclaimed the history entry the navigation had just pushed. Rounds three and four stopped finding defects in the epic and started finding them in the review's own fix commits — a test named after a branch it never reached, comments claiming more than the code does — which is the signal to stop.
|
||||
- Two process notes worth keeping. A commit in this epic fixed a bug that does not exist: its message and two comments claimed a sort key raises when two rows both lack a timestamp, which tuple comparison never reaches — three verifiers executed it independently, and the rationale is now corrected in place rather than quietly deleted. And three of the regression tests written during the fix rounds could not go red for the defect they named until they were rewritten, which is why every test added in a fix round on this epic was mutation-checked.
|
||||
|
||||
## 0.82.0 — 2026-09-06
|
||||
|
||||
Siftlode gets a keyboard. There is one place that lists every shortcut, a jump-to box that takes you anywhere by name, and a right-click menu on playlist rows — plus the whole app now holds together at a tablet's width and stops thinning itself out on a very wide screen.
|
||||
|
||||
@@ -58,13 +58,14 @@ how it gets built. Concretely:
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`,
|
||||
`security`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
`security`, `logs`, `logsink`, `diagnostics`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||
its menus, `Modal`/`Overlay`, `Button`/`form`, `DataTable`/`VirtualGrid`, `QueryState`, …). That
|
||||
boundary is one-way and **lint-enforced**, not just described: a platform component may not import a
|
||||
feature component or a domain API slice (`no-restricted-imports` in `eslint.config.js`, with its own
|
||||
test). A new generic component goes in `ui/`; see [`frontend/src/components/ui/README.md`](frontend/src/components/ui/README.md).
|
||||
feature component or a domain API slice, and one layer down `lib/` may not import from `components/` at
|
||||
all, not even a type (`no-restricted-imports` in `eslint.config.js`, with its own test). A new generic
|
||||
component goes in `ui/`; see [`frontend/src/components/ui/README.md`](frontend/src/components/ui/README.md).
|
||||
Data via react-query (`qk` factory), per-account `localStorage`, and server prefs; API through
|
||||
`lib/api/*` slices over `core.req<T>`.
|
||||
|
||||
@@ -89,8 +90,13 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE action button | `ui/Button` — pick a `SIZES` rung | a hand-rolled `<button className="px-3 py-2 …">` |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE control | the chip/segmented primitives (`Select` only for >6 options) | a native `<select>` |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
|
||||
## House rules that bite (learned from real bugs)
|
||||
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN adduser --disabled-password --gecos "" appuser && chown -R appuser /app
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["sh", "entrypoint.sh"]
|
||||
@@ -1,6 +1,8 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
# Split the path list the way the OS does (alembic's legacy space/comma/colon split is deprecated).
|
||||
path_separator = os
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from app.config import settings
|
||||
from app.db import Base
|
||||
from app.logs import configure_logging
|
||||
import app.models # noqa: F401 (import so models register on Base.metadata)
|
||||
|
||||
config = context.config
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
# Migrations run at container start (entrypoint.sh), so their output is part of the API's log
|
||||
# stream — one format with everything else, not alembic.ini's own fileConfig shape.
|
||||
configure_logging("alembic")
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""comfort_pins — the shelf you fall back to when you cannot decide
|
||||
|
||||
Revision ID: 0083_comfort_pins
|
||||
Revises: 0082_invite_lang
|
||||
Create Date: 2026-09-06
|
||||
|
||||
E24's one new table. The shelf holds five different things — a video, a YouTube channel, a Plex show,
|
||||
a Plex movie, a playlist — because comfort media is usually "put on X", a series or a channel, and
|
||||
almost never one particular video.
|
||||
|
||||
Three shapes were possible and two are wrong here:
|
||||
|
||||
- a boolean on `video_states` / `plex_states` cannot ORDER a shelf, and has nowhere to put a channel
|
||||
or a show (neither has a per-user state row at all);
|
||||
- a polymorphic `(kind, ref)` string pair orders and holds anything, but drops the foreign key — and a
|
||||
Plex item removed from the library, or a channel the user unsubscribes and prunes, would then leave
|
||||
a dead tile on the one surface whose entire job is to be reliable when nothing else is.
|
||||
|
||||
So: one nullable CASCADE-ing FK per kind, plus `num_nonnulls(...) = 1`. The union invariant lives in
|
||||
the schema rather than in a comment at every insert site, which is the same rule that the
|
||||
`cancelled`/`canceled` bug taught this codebase.
|
||||
|
||||
NULLs are DISTINCT in a Postgres unique index, so the five per-kind `(user_id, <ref>)` uniques each
|
||||
stop a double-pin without colliding with the other four kinds' NULL rows.
|
||||
|
||||
No backfill: an empty shelf is the honest state for every existing account. The surface renders its
|
||||
own empty state and invites the first pin.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0083_comfort_pins"
|
||||
down_revision: str | None = "0082_invite_lang"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"comfort_pins",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("video_id", sa.String(length=16), sa.ForeignKey("videos.id", ondelete="CASCADE")),
|
||||
sa.Column(
|
||||
"channel_id", sa.String(length=32), sa.ForeignKey("channels.id", ondelete="CASCADE")
|
||||
),
|
||||
sa.Column("plex_item_id", sa.Integer(), sa.ForeignKey("plex_items.id", ondelete="CASCADE")),
|
||||
sa.Column("plex_show_id", sa.Integer(), sa.ForeignKey("plex_shows.id", ondelete="CASCADE")),
|
||||
sa.Column("playlist_id", sa.Integer(), sa.ForeignKey("playlists.id", ondelete="CASCADE")),
|
||||
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"num_nonnulls(video_id, channel_id, plex_item_id, plex_show_id, playlist_id) = 1",
|
||||
name="ck_comfort_pin_one_ref",
|
||||
),
|
||||
sa.UniqueConstraint("user_id", "video_id", name="uq_comfort_pin_video"),
|
||||
sa.UniqueConstraint("user_id", "channel_id", name="uq_comfort_pin_channel"),
|
||||
sa.UniqueConstraint("user_id", "plex_item_id", name="uq_comfort_pin_plex_item"),
|
||||
sa.UniqueConstraint("user_id", "plex_show_id", name="uq_comfort_pin_plex_show"),
|
||||
sa.UniqueConstraint("user_id", "playlist_id", name="uq_comfort_pin_playlist"),
|
||||
)
|
||||
op.create_index("ix_comfort_pins_user_id", "comfort_pins", ["user_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_comfort_pins_user_id", table_name="comfort_pins")
|
||||
op.drop_table("comfort_pins")
|
||||
@@ -0,0 +1,41 @@
|
||||
"""add download_jobs.error_detail — the domain-specific entry of the failure catalog (E11.5 S1)
|
||||
|
||||
Revision ID: 0084_dl_job_error_detail
|
||||
Revises: 0083_comfort_pins
|
||||
Create Date: 2026-09-13
|
||||
|
||||
`error_code` is the generic category (it decides whether a Retry is offered). `error_detail` names
|
||||
the host-specific catalog entry `downloads.errors.classify` matched on top of it — `facebook.login_wall`
|
||||
and its siblings — so the UI can say what went wrong ON THAT SITE and what to do about it, instead of
|
||||
the category's generic sentence. Nullable: most failures have no host-specific entry.
|
||||
|
||||
Backfill: a job's code is written once, at failure time, so a new rule never reaches the rows that
|
||||
failed before it. The one shape measured on prod (2026-09-13, jobs 9/66/75/145/156/157/182/256/257) is
|
||||
yt-dlp's `[facebook] <id>: Cannot parse data`, stored as `unknown`. Without a cookie that is Facebook's
|
||||
login wall. Whether a cookie was attached at the time is not recorded anywhere, so every old row gets
|
||||
the no-cookie entry — on prod no account had a Facebook cookie, so that is what they all were.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0084_dl_job_error_detail"
|
||||
down_revision: str | None = "0083_comfort_pins"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("download_jobs", sa.Column("error_detail", sa.String(length=48), nullable=True))
|
||||
op.execute(
|
||||
sa.text(
|
||||
"UPDATE download_jobs SET error_code = 'login_required', "
|
||||
"error_detail = 'facebook.login_wall' "
|
||||
"WHERE (error_code IS NULL OR error_code = 'unknown') "
|
||||
"AND error ILIKE '%[facebook]%cannot parse data%'"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("download_jobs", "error_detail")
|
||||
@@ -0,0 +1,26 @@
|
||||
"""add users.last_login_at — when the account last signed in (E22 S1)
|
||||
|
||||
Revision ID: 0085_users_last_login_at
|
||||
Revises: 0084_dl_job_error_detail
|
||||
Create Date: 2026-09-16
|
||||
|
||||
Stamped by the one sign-in path (`routes/auth._establish_session`), which the Google callback and
|
||||
password login share. No backfill is possible: the signal exists only at sign-in, so existing rows
|
||||
read NULL until each account signs in again, and the admin UI says so rather than inventing a date.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0085_users_last_login_at"
|
||||
down_revision: str | None = "0084_dl_job_error_detail"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("users", sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "last_login_at")
|
||||
@@ -0,0 +1,48 @@
|
||||
"""add log_events — stored log events for diagnostics (E22 S3)
|
||||
|
||||
Revision ID: 0086_log_events
|
||||
Revises: 0085_users_last_login_at
|
||||
Create Date: 2026-09-17
|
||||
|
||||
Written by app/logsink.py's background writer (WARNING and above, request lines), pruned by the log_gc
|
||||
job after `log_retention_days`. Sized against prod on 2026-09-16: ~2k rows on a normal day, ~14k on a
|
||||
YouTube-RSS-outage day before flood control — a few MB over the 7-day window. `user_id` carries no
|
||||
foreign key on purpose (see models.LogEvent).
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0086_log_events"
|
||||
down_revision: str | None = "0085_users_last_login_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"log_events",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True),
|
||||
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("level", sa.String(10), nullable=False),
|
||||
sa.Column("sev", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("source", sa.String(16), nullable=False),
|
||||
sa.Column("logger", sa.String(128), nullable=False),
|
||||
sa.Column("event", sa.String(64), nullable=True),
|
||||
sa.Column("msg", sa.Text(), nullable=False),
|
||||
sa.Column("attrs", postgresql.JSONB(), nullable=True),
|
||||
sa.Column("exc", sa.Text(), nullable=True),
|
||||
sa.Column("request_id", sa.String(64), nullable=True),
|
||||
sa.Column("user_id", sa.Integer(), nullable=True),
|
||||
sa.Column("debug_session_id", sa.String(64), nullable=True),
|
||||
sa.Column("job_id", sa.Integer(), nullable=True),
|
||||
sa.Column("version", sa.String(64), nullable=True),
|
||||
)
|
||||
op.create_index("ix_log_events_ts", "log_events", ["ts"])
|
||||
op.create_index("ix_log_events_request_id", "log_events", ["request_id"])
|
||||
op.create_index("ix_log_events_user_ts", "log_events", ["user_id", "ts"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("log_events")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add users.diagnostics_level + diagnostics_until — a user's consented diagnostics level (E22 S5)
|
||||
|
||||
Revision ID: 0087_users_diagnostics_level
|
||||
Revises: 0086_log_events
|
||||
Create Date: 2026-09-17
|
||||
|
||||
Both NULL = the base level (Alap). A raised level is ALWAYS paired with its expiry, computed by the
|
||||
server at consent time (`app/diagnostics.py`), so no client can ask for a level that never ends.
|
||||
Columns rather than `users.preferences`: that JSON is written wholesale by the client, and a consent
|
||||
with a lifetime must not be something a client can write.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0087_users_diagnostics_level"
|
||||
down_revision: str | None = "0086_log_events"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("users", sa.Column("diagnostics_level", sa.String(8), nullable=True))
|
||||
op.add_column(
|
||||
"users", sa.Column("diagnostics_until", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "diagnostics_until")
|
||||
op.drop_column("users", "diagnostics_level")
|
||||
@@ -20,9 +20,10 @@ Three layers, by *who* runs the code — not by feature:
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline.
|
||||
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, … are single-file primitives many
|
||||
layers import. They depend on nothing above them (a core module must never import a `routes/…`
|
||||
module).
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table) and `diagnostics` (a user's consented log
|
||||
level), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
module must never import a `routes/…` module).
|
||||
|
||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||
*second* caller needs it (a worker as well as a route; a scheduler as well as a route), extract it down
|
||||
|
||||
+72
-18
@@ -6,12 +6,14 @@ from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
from app import diagnostics
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
from app.models import Invite, LogEvent, User
|
||||
from app.security import decrypt_optional
|
||||
from app.utils import canonical_email
|
||||
|
||||
@@ -245,6 +247,12 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
||||
)
|
||||
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
|
||||
db.execute(delete(User).where(User.id == user_id))
|
||||
# Stored log events name the account by id with no foreign key to cascade from (models.LogEvent),
|
||||
# so they go explicitly — they are request history, i.e. personal data. Forgotten FIRST: this very
|
||||
# request's access line, a queued row, a later background task would otherwise put rows back
|
||||
# under the id after this DELETE (logs.forget_user).
|
||||
logs.forget_user(user_id)
|
||||
db.execute(delete(LogEvent).where(LogEvent.user_id == user_id))
|
||||
# Every canonically-equal invite goes, not just the literal spelling: is_allowed matches on the
|
||||
# canonical form, so leaving a sibling row behind would keep granting access to a mailbox whose
|
||||
# account was just erased — and prod really does carry approved pairs for one mailbox.
|
||||
@@ -269,21 +277,29 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
||||
ACTIVE_ACCOUNT_HEADER = "x-siftlode-account"
|
||||
|
||||
|
||||
def resolved_user_id(conn: HTTPConnection) -> tuple[int | None, bool]:
|
||||
def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int | None, bool]:
|
||||
"""Which account this connection acts as, and whether that's the session's default account.
|
||||
Takes any HTTPConnection — an HTTP Request OR a WebSocket — so the WS push channel shares this
|
||||
exact per-tab resolution instead of re-implementing it.
|
||||
|
||||
The signed cookie holds the *wallet* (`account_ids`, every account signed into this browser)
|
||||
plus a default `user_id`. A caller may override the default with the X-Siftlode-Account header,
|
||||
but ONLY for an account already in the wallet — so a tab can't impersonate an account that never
|
||||
authenticated here. Everything else (WebSocket, plain navigations) keeps using the cookie default.
|
||||
plus a default `user_id`. A caller may name its account with the X-Siftlode-Account header (or
|
||||
`?account=`), but ONLY an account in the wallet — so a tab can't impersonate an account that
|
||||
never authenticated here. A connection that names no account acts as the cookie default.
|
||||
|
||||
A connection naming an account NOT in the wallet acts as nobody (`None`) — typically a tab still
|
||||
pinned to an account that was signed out in another tab. It must not fall back to the default:
|
||||
that tab would go on acting as an account it never showed, its player checkpoints and pref saves
|
||||
landing there and its Sign out signing THAT account out. Only the bootstrap /api/me probe passes
|
||||
`strict=False` and gets the default, which is how such a tab learns who it now is (the frontend
|
||||
re-pins it and reloads). An account whose session ended while it stayed in the wallet (deleted,
|
||||
suspended, revoked) leaves the wallet when it is refused (`_session_user`), so a tab pinned to it
|
||||
takes the same way back.
|
||||
"""
|
||||
default_id = conn.session.get("user_id")
|
||||
wallet = conn.session.get("account_ids") or []
|
||||
# Header for normal XHR; ?account= for contexts that can't set headers (WebSocket, and a
|
||||
# plain <a> file download). Both are wallet-gated below, so neither can impersonate an
|
||||
# account that never signed into this browser.
|
||||
# plain <a> file download). Both are wallet-gated below.
|
||||
hdr = conn.headers.get(ACTIVE_ACCOUNT_HEADER) or conn.query_params.get("account")
|
||||
if hdr:
|
||||
try:
|
||||
@@ -292,28 +308,46 @@ def resolved_user_id(conn: HTTPConnection) -> tuple[int | None, bool]:
|
||||
hid = None
|
||||
if hid is not None and hid in wallet:
|
||||
return hid, hid == default_id
|
||||
if strict:
|
||||
return None, False
|
||||
return default_id, True
|
||||
|
||||
|
||||
def drop_from_wallet(session: dict, user_id: int) -> None:
|
||||
"""Take one account out of this browser's wallet (`account_ids`); the session default is untouched."""
|
||||
session["account_ids"] = [a for a in (session.get("account_ids") or []) if a != user_id]
|
||||
|
||||
|
||||
def _end_account_session(request: Request, user_id: int, is_default: bool) -> None:
|
||||
"""An account whose session here is no longer valid. Only nuke the whole session when it is the
|
||||
SESSION DEFAULT; a per-tab account just leaves the wallet (other tabs' default may still be valid).
|
||||
Leaving is what lets a tab pinned to it recover: its next /api/me names an account no longer signed
|
||||
in here and gets the default. Left in, it answered "signed out" — the landing page, with the default
|
||||
still signed in, and no way off it but closing the tab."""
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
else:
|
||||
drop_from_wallet(request.session, user_id)
|
||||
|
||||
|
||||
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
user_id, is_default = resolved_user_id(request)
|
||||
return _session_user(request, db, *resolved_user_id(request))
|
||||
|
||||
|
||||
def _session_user(request: Request, db: Session, user_id: int | None, is_default: bool) -> User:
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
user = db.get(User, user_id)
|
||||
if user is None or not user.is_active or user.is_suspended:
|
||||
# Suspended/deactivated mid-session → drop the session so the block takes effect at once.
|
||||
# But only nuke the whole session when the SESSION DEFAULT identity is gone; a stale
|
||||
# per-tab header account just 401s that one tab (other tabs' default may still be valid).
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
# Suspended/deactivated mid-session → end it so the block takes effect at once.
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
# SA4: reject a cookie whose recorded epoch is behind the account's current one (revoked by a
|
||||
# password reset/change or "log out everywhere"). A cookie with no recorded epoch (pre-SA4, or an
|
||||
# account signed in before this shipped) is treated as epoch 0 — so the FIRST bump revokes it too.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(user_id), 0)) != (user.session_epoch or 0):
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
# Always keep the session default in the switchable list — covers sessions created before
|
||||
# multi-session existed (their account_ids was never seeded), so the switcher isn't empty.
|
||||
@@ -322,15 +356,31 @@ def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
if default_id and default_id not in accounts:
|
||||
accounts.append(default_id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
# Attribute this request's log lines to the account (E22). Here, not in the request middleware:
|
||||
# this is the one place a session becomes an account, and the middleware runs outside both the
|
||||
# session middleware and this task's context. `request.state` is shared across middleware layers
|
||||
# (it lives on the ASGI scope), so the access-log line can read it back. The user's consented
|
||||
# diagnostics level rides along: their requests are logged — and stored — in that detail (S5).
|
||||
logs.set_user(user.id, diagnostics.effective_level(user), user.diagnostics_until)
|
||||
request.state.user_id = user.id
|
||||
return user
|
||||
|
||||
|
||||
def optional_current_user(request: Request, db: Session = Depends(get_db)) -> User | None:
|
||||
"""Like `current_user` but returns None instead of raising 401 when there's no valid session.
|
||||
Used by the bootstrap /api/me probe so the logged-out landing doesn't log a failed request to
|
||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it)."""
|
||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it).
|
||||
The one lenient resolution (`strict=False`): a tab naming an account no longer signed in here gets
|
||||
the default account, which is how it learns who it now is — see `resolved_user_id`. That includes an
|
||||
account refused right here, which `_session_user` has just taken out of the wallet."""
|
||||
user_id, is_default = resolved_user_id(request, strict=False)
|
||||
try:
|
||||
return current_user(request, db)
|
||||
return _session_user(request, db, user_id, is_default)
|
||||
except HTTPException:
|
||||
if is_default:
|
||||
return None
|
||||
try:
|
||||
return _session_user(request, db, *resolved_user_id(request, strict=False))
|
||||
except HTTPException:
|
||||
return None
|
||||
|
||||
@@ -341,7 +391,11 @@ def require_human(user: User = Depends(current_user)) -> User:
|
||||
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
|
||||
where there's no scope check to lean on."""
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="Not available in the demo account.")
|
||||
# Structured, because `req` leaves 403 to the caller and `notifyActionError` only shows a
|
||||
# STRUCTURED detail — a plain string here reached the demo user as their caller's generic
|
||||
# "couldn't do that", with no hint that the demo account is the reason (round 6). Affordances
|
||||
# this gate sits behind are not all UI-gated on `is_demo` (the comfort shelf is not).
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
return user
|
||||
|
||||
|
||||
|
||||
+70
-6
@@ -1,6 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args, get_origin
|
||||
|
||||
from pydantic import model_validator
|
||||
from pydantic import ValidationInfo, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Shipped placeholder; production must override it. Used by the startup guard below.
|
||||
@@ -20,6 +21,33 @@ def _version_from_file() -> str:
|
||||
return "dev"
|
||||
|
||||
|
||||
# Closed value sets. A setting typed with one of these is a CHOICE: pydantic rejects anything else at
|
||||
# boot, and sysconfig derives the admin picker's options from the same Literal — so the list lives in
|
||||
# exactly one place and a consumer never has to guess what a typo should fall back to.
|
||||
SearchSource = Literal["scrape", "api"]
|
||||
DownloadLayout = Literal["plex", "flat"]
|
||||
LogLevel = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR"]
|
||||
LogFormat = Literal["json", "text"]
|
||||
TranscodeHw = Literal["auto", "on", "off"]
|
||||
# libx264's presets, fastest first (`placebo` left out on purpose: no measurable gain for its cost).
|
||||
TranscodePreset = Literal[
|
||||
"ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"
|
||||
]
|
||||
|
||||
|
||||
def choices_of(annotation: object) -> tuple[str, ...]:
|
||||
"""The allowed values of a choice-typed setting, or () for any other type."""
|
||||
return tuple(get_args(annotation)) if get_origin(annotation) is Literal else ()
|
||||
|
||||
|
||||
def canonical_choice(options: tuple[str, ...], raw: object) -> str | None:
|
||||
"""The option `raw` names, in its canonical spelling (any letter case, surrounding space
|
||||
ignored), or None. The ONE matching rule for a choice value from the env and from the admin
|
||||
API, so a spelling one accepts the other cannot refuse."""
|
||||
folded = str(raw).strip().casefold()
|
||||
return next((option for option in options if option.casefold() == folded), None)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
@@ -57,7 +85,7 @@ class Settings(BaseSettings):
|
||||
# VPS Caddy front door. A request from any other peer (someone hitting the app port directly)
|
||||
# is NOT trusted and its raw socket IP is used, so it can't spoof its rate-limit identity via
|
||||
# XFF. Empty = trust no proxy (use the direct peer) — correct for a directly-exposed / local-dev
|
||||
# deploy. Comma-separated. See routes.auth._client_ip + the deploy docs.
|
||||
# deploy. Comma-separated. See ratelimit.client_key + the deploy docs.
|
||||
trusted_proxy_ips: str = ""
|
||||
|
||||
# --- Outbound email (onboarding: access-request + approval notices) ---
|
||||
@@ -85,7 +113,7 @@ class Settings(BaseSettings):
|
||||
search_daily_limit_per_user: int = 70
|
||||
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
|
||||
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
|
||||
search_source: str = "scrape"
|
||||
search_source: SearchSource = "scrape"
|
||||
# Recent-first backfill: how far back to fetch on the first pass per channel.
|
||||
backfill_recent_max_videos: int = 100
|
||||
backfill_recent_max_days: int = 365
|
||||
@@ -155,6 +183,9 @@ class Settings(BaseSettings):
|
||||
# per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
|
||||
# worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
|
||||
# Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
|
||||
# A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients
|
||||
# of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there.
|
||||
# See adapters.YouTubeAdapter._COOKIE_CAPABLE.
|
||||
download_player_clients: str = "android_vr"
|
||||
download_player_clients_fallback: str = "web_safari,web,android"
|
||||
|
||||
@@ -221,10 +252,27 @@ class Settings(BaseSettings):
|
||||
audit_retention_days: int = 180
|
||||
audit_gc_minutes: int = 1440
|
||||
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
||||
download_layout: str = "plex"
|
||||
download_layout: DownloadLayout = "plex"
|
||||
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
||||
sponsorblock_default: bool = False
|
||||
|
||||
# --- Diagnostics / logging (E22) ---
|
||||
# BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at
|
||||
# startup and on change without a restart. This is what a process logs at before it can reach
|
||||
# the database — and the only knob the worker/runner children read, since they are short-lived.
|
||||
# TRACE (below DEBUG) is the "Teljes" diagnostics level; see app/logs.py.
|
||||
log_level: LogLevel = "INFO"
|
||||
# "json" (one object per line — what Loki and the log_events sink ingest) or "text" (a readable
|
||||
# line for a local terminal). Redaction applies to both.
|
||||
log_format: LogFormat = "json"
|
||||
# Also write events to the `log_events` table (app/logsink.py): WARNING and above, the request
|
||||
# lines, and — once they exist — client batches. Off only where no table is wanted (the tests).
|
||||
log_db_sink: bool = True
|
||||
# Days a `log_events` row is kept (the privacy policy promises this), and how often the log_gc job
|
||||
# enforces it — hourly, so a row never outlives the promise by more than an hour.
|
||||
log_retention_days: int = 7
|
||||
log_gc_minutes: int = 60
|
||||
|
||||
# --- Plex integration (optional) ---
|
||||
# The Plex module plays the LOCAL physical media file directly (our own ffmpeg/direct-serve);
|
||||
# Plex is used only for metadata + optional watch-state sync. Requires the Plex server to be
|
||||
@@ -266,12 +314,17 @@ class Settings(BaseSettings):
|
||||
# Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the
|
||||
# source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render
|
||||
# node exists (skip the decodability gate); "off" = always software. See plex/stream.py `_select_encode`.
|
||||
plex_transcode_hw: str = "auto"
|
||||
plex_transcode_hw: TranscodeHw = "auto"
|
||||
# Software-fallback libx264 tuning (IGNORED on the VAAPI path). `preset` trades CPU for size; `crf`
|
||||
# trades quality for size (lower = better + bigger). `max_height` caps the SOFTWARE transcode's output
|
||||
# height (0 = keep source) so a CPU re-encode of a tall source stays affordable — the GPU path always
|
||||
# encodes at source resolution.
|
||||
plex_transcode_preset: str = "veryfast"
|
||||
# Loudness normalisation of the downmixed stereo audio on the HLS path (EBU R128 `loudnorm`).
|
||||
# OFF by default: it makes a film ~11 dB louder and narrows its dynamic range, which is a taste
|
||||
# call, not a fix. Only affects files that go through remux/transcode — a direct-played file is
|
||||
# served untouched, but those are browser-native stereo anyway, not the 5.1 mixes this is for.
|
||||
plex_audio_loudnorm: bool = False
|
||||
plex_transcode_preset: TranscodePreset = "veryfast"
|
||||
plex_transcode_crf: int = 20
|
||||
plex_transcode_max_height: int = 1080
|
||||
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
|
||||
@@ -320,6 +373,17 @@ class Settings(BaseSettings):
|
||||
https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev."""
|
||||
return self.oauth_redirect_url.lower().startswith("https://")
|
||||
|
||||
@field_validator("*", mode="before")
|
||||
@classmethod
|
||||
def _canonical_choice(cls, value: object, info: ValidationInfo) -> object:
|
||||
"""A choice setting accepts its value in any letter case (`LOG_LEVEL=debug`) and stores the
|
||||
canonical spelling; an unknown value still fails the boot with pydantic's own message."""
|
||||
field = cls.model_fields.get(info.field_name or "")
|
||||
options = choices_of(field.annotation) if field else ()
|
||||
if options and isinstance(value, str):
|
||||
return canonical_choice(options, value) or value
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _enforce_production_secrets(self) -> "Settings":
|
||||
"""Fail fast rather than boot a public instance with the shipped placeholder
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""A user's diagnostics level (E22 S5): what they consented to, and for how long.
|
||||
|
||||
Three levels, named in the UI Alap / Részletes / Teljes:
|
||||
|
||||
* INFO — the base, for everyone, no consent. The browser keeps its log in memory; nothing extra
|
||||
is sent or stored.
|
||||
* DEBUG — consented. The server logs this user's requests in detail and stores every line of
|
||||
them, and the browser streams its own log to `POST /api/diagnostics/events`.
|
||||
* TRACE — consented. As DEBUG, plus what the user typed into a search.
|
||||
|
||||
A raised level always lapses after `ELEVATION_HOURS`: the expiry is computed HERE at consent time,
|
||||
never taken from the client, and an expired level reads as INFO without anything having to clear it.
|
||||
The shared demo account cannot raise its level — its log would mix every visitor's session.
|
||||
|
||||
Core layer: imports `models` and `sysconfig`.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Literal, get_args
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.models import User
|
||||
|
||||
DiagnosticsLevel = Literal["INFO", "DEBUG", "TRACE"]
|
||||
LEVELS: tuple[str, ...] = get_args(DiagnosticsLevel)
|
||||
BASE: DiagnosticsLevel = "INFO"
|
||||
ELEVATION_HOURS = 24
|
||||
|
||||
|
||||
def effective_level(user: User, now: datetime | None = None) -> DiagnosticsLevel:
|
||||
"""The level in force for `user` right now: the consented one until it expires, else BASE."""
|
||||
level, until = user.diagnostics_level, user.diagnostics_until
|
||||
if level not in LEVELS or level == BASE or until is None:
|
||||
return BASE
|
||||
if until <= (now or datetime.now(timezone.utc)):
|
||||
return BASE
|
||||
return level # type: ignore[return-value] — narrowed by the LEVELS check above
|
||||
|
||||
|
||||
def state(db: Session, user: User) -> dict:
|
||||
"""The `/api/me` view of it: the level in force, when a raised one lapses, and how long stored
|
||||
log lines are kept — the consent text states that number, so it must be the instance's own."""
|
||||
level = effective_level(user)
|
||||
return {
|
||||
"level": level,
|
||||
"until": user.diagnostics_until.isoformat() if level != BASE else None,
|
||||
"retention_days": sysconfig.get_int(db, "log_retention_days"),
|
||||
}
|
||||
|
||||
|
||||
def set_level(user: User, level: DiagnosticsLevel) -> None:
|
||||
"""Record a consent (or its withdrawal). The caller commits. Raising the level again restarts
|
||||
the clock — each consent is for the next ELEVATION_HOURS, not an extension of an older one."""
|
||||
if level == BASE:
|
||||
user.diagnostics_level = None
|
||||
user.diagnostics_until = None
|
||||
return
|
||||
user.diagnostics_level = level
|
||||
user.diagnostics_until = datetime.now(timezone.utc) + timedelta(hours=ELEVATION_HOURS)
|
||||
@@ -0,0 +1,10 @@
|
||||
"""E24 — the read-model behind the Home surface ("Mit nézzek most?").
|
||||
|
||||
The anti-paralysis answer layer: one hero suggestion, a cross-source resume rail, and the user's own
|
||||
comfort shelf, all served from our own Postgres with no upstream call at serve time (the same rule
|
||||
the Plex Discover surface follows).
|
||||
|
||||
Layering, mirroring `app/plex/`: `budget` (the time bands) and `cards` (the unified row shape) are
|
||||
pure; `queries` holds the SQL; `rails`, `pins` and `hero` compose. Nothing here imports from
|
||||
`app.routes` — `routes/discover.py` is a thin HTTP shell over these.
|
||||
"""
|
||||
@@ -0,0 +1,62 @@
|
||||
"""The time budget — E24's replacement for "what mood are you in?".
|
||||
|
||||
`random-gondolatok` 15/D asked for a mood questionnaire before the recommendations. The same file's
|
||||
later BA-interview note (#23) says the mood half is not what makes this work: the friction is
|
||||
deciding at all. A mood→genre mapping would also be invented data — we store no mood signal and never
|
||||
will — whereas "how much time do you have" is a question the library can actually answer, from a
|
||||
column we already have.
|
||||
|
||||
So the surface asks about TIME, and the answer is deterministic: four bands, no model, no latency.
|
||||
The band also picks the source for free — ten minutes is a YouTube video, an evening is a movie —
|
||||
which is exactly the cross-source behaviour the Home surface exists to provide.
|
||||
|
||||
The bands overlap on purpose. "I have 30 minutes" should not be answered with a two-minute Short just
|
||||
because it fits; a lower bound is as much a part of the promise as the upper one.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
Budget = Literal["short", "medium", "long", "any"]
|
||||
|
||||
DEFAULT_BUDGET: Budget = "any"
|
||||
|
||||
# budget -> (min_seconds, max_seconds); None = unbounded on that side.
|
||||
BANDS: dict[str, tuple[int | None, int | None]] = {
|
||||
"short": (None, 12 * 60), # "10 perc" — a bit of slack over the round number the chip shows
|
||||
"medium": (8 * 60, 40 * 60), # "30 perc"
|
||||
"long": (40 * 60, None), # "Egy este"
|
||||
"any": (None, None), # "Mindegy"
|
||||
}
|
||||
|
||||
|
||||
def band(budget: str) -> tuple[int | None, int | None]:
|
||||
"""The (min, max) seconds for a budget. Unknown budgets fall back to the unbounded band rather
|
||||
than raising: the route validates the vocabulary with a Literal, so anything reaching here with a
|
||||
bad value is our own bug, and an unfiltered shelf is a better failure than a 500 on the landing
|
||||
page."""
|
||||
return BANDS.get(budget, BANDS["any"])
|
||||
|
||||
|
||||
def remaining_seconds(duration_seconds: int | None, position_seconds: int) -> int | None:
|
||||
"""What is actually LEFT to watch. The question is "how much time do you have", so a 90-minute
|
||||
film with 8 minutes to go belongs in the ten-minute band, not the evening one. Unknown duration
|
||||
stays unknown — see `fits`."""
|
||||
if duration_seconds is None:
|
||||
return None
|
||||
return max(0, duration_seconds - max(0, position_seconds))
|
||||
|
||||
|
||||
def fits(seconds: int | None, budget: str) -> bool:
|
||||
"""Does a runtime fit the budget?
|
||||
|
||||
An unknown duration fits only the unbounded band. We cannot promise an item finishes inside ten
|
||||
minutes when we do not know how long it is, and quietly including it would make the one control
|
||||
on this page dishonest — the failure mode the house rules call an un-honest state."""
|
||||
lo, hi = band(budget)
|
||||
if seconds is None:
|
||||
return lo is None and hi is None
|
||||
if lo is not None and seconds < lo:
|
||||
return False
|
||||
if hi is not None and seconds > hi:
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,141 @@
|
||||
"""The unified discover card: one row shape the Home surface can render without knowing which
|
||||
module it came from.
|
||||
|
||||
This is deliberately a THIN mapping, not a new serializer layer. The per-source projections already
|
||||
exist and are already the shapes the frontend's `VideoCard` and `PlexPosterCard` consume —
|
||||
`feedquery.serialize` for YouTube, `plex.cards.movie_card`/`show_card`/`leaf_card` for Plex — so each
|
||||
unified card carries the native payload under `video` / `plex` alongside the handful of fields the
|
||||
rail chrome itself needs (title, thumbnail, progress, runtime). The Home page therefore renders with
|
||||
the existing card components rather than a third one, and a field added to a feed row or a Plex card
|
||||
still gets added in exactly one place. CLAUDE.md's "not wanted" list names a base serializer for
|
||||
precisely this reason.
|
||||
|
||||
`aspect` is the one genuinely new field. A rail has to know whether a tile is a 16:9 still or a 2:3
|
||||
poster, and that is a property of the ARTWORK, not of the module: a Plex episode ships a landscape
|
||||
still exactly like a YouTube video does, while a movie and a show ship posters. Emitting it lets the
|
||||
rail layer stay domain-free instead of branching on `source`.
|
||||
|
||||
`subtitle` only ever carries DATA — a channel name, a show title, a year. Nothing here composes an
|
||||
English phrase: this repo is HU+EN with zero key drift, and a sentence assembled in the backend is a
|
||||
string no translation file can reach. Anything that needs wording (season counts, "N videó") is left
|
||||
to the client, which has both the native payload and the translations.
|
||||
"""
|
||||
|
||||
from app.discover.budget import remaining_seconds
|
||||
from app.models import Channel, Playlist
|
||||
|
||||
# Plex artwork is a poster for the things you browse and a landscape still for the things you play.
|
||||
_POSTER_KINDS = ("movie", "show")
|
||||
|
||||
|
||||
def _base(
|
||||
*,
|
||||
source: str, # "youtube" | "plex"
|
||||
kind: str, # "video" | "movie" | "show" | "episode" | "channel" | "playlist"
|
||||
id: str,
|
||||
title: str | None,
|
||||
aspect: str, # "wide" (16:9) | "poster" (2:3)
|
||||
subtitle: str | None = None,
|
||||
thumb: str | None = None,
|
||||
duration_seconds: int | None = None,
|
||||
position_seconds: int = 0,
|
||||
status: str = "new",
|
||||
) -> dict:
|
||||
return {
|
||||
"source": source,
|
||||
"kind": kind,
|
||||
"id": id,
|
||||
"title": title or "",
|
||||
"subtitle": subtitle,
|
||||
"thumb": thumb,
|
||||
"aspect": aspect,
|
||||
"duration_seconds": duration_seconds,
|
||||
"position_seconds": position_seconds,
|
||||
"remaining_seconds": remaining_seconds(duration_seconds, position_seconds),
|
||||
"status": status,
|
||||
# Native payloads — at most one is ever set. The client opens the player from these.
|
||||
"video": None,
|
||||
"plex": None,
|
||||
}
|
||||
|
||||
|
||||
def key(card: dict) -> str:
|
||||
"""The identity of a card ACROSS sources — `<source>:<id>`. A Plex rating_key and a YouTube video
|
||||
id could in principle collide as bare strings, and the reroll's exclusion list has to be able to
|
||||
say "not that one" about either."""
|
||||
return f"{card['source']}:{card['id']}"
|
||||
|
||||
|
||||
def from_video(row: dict) -> dict:
|
||||
"""`row` is a `feedquery.serialize` dict — the same object the feed grid renders."""
|
||||
card = _base(
|
||||
source="youtube",
|
||||
kind="video",
|
||||
id=row["id"],
|
||||
title=row["title"],
|
||||
aspect="wide",
|
||||
subtitle=row.get("channel_title"),
|
||||
thumb=row.get("thumbnail_url"),
|
||||
duration_seconds=row.get("duration_seconds"),
|
||||
position_seconds=row.get("position_seconds") or 0,
|
||||
status=row.get("status") or "new",
|
||||
)
|
||||
card["video"] = row
|
||||
return card
|
||||
|
||||
|
||||
def from_plex(row: dict) -> dict:
|
||||
"""`row` is a `plex.cards` dict (movie / show / leaf). Its own `type` is the kind."""
|
||||
kind = row.get("type") or "movie"
|
||||
if kind == "episode":
|
||||
subtitle = row.get("show_title")
|
||||
elif kind == "show":
|
||||
subtitle = None # season count lives in the native payload; the client words it
|
||||
else:
|
||||
subtitle = str(row["year"]) if row.get("year") else None
|
||||
|
||||
card = _base(
|
||||
source="plex",
|
||||
kind=kind,
|
||||
id=row["id"],
|
||||
title=row.get("title"),
|
||||
aspect="poster" if kind in _POSTER_KINDS else "wide",
|
||||
subtitle=subtitle,
|
||||
thumb=row.get("thumb"),
|
||||
duration_seconds=row.get("duration_seconds"),
|
||||
position_seconds=row.get("position_seconds") or 0,
|
||||
status=row.get("status") or "new",
|
||||
)
|
||||
card["plex"] = row
|
||||
return card
|
||||
|
||||
|
||||
def from_channel(ch: Channel, unwatched: int) -> dict:
|
||||
"""A pinned channel. It is not itself playable — the shelf resolves it to its newest unwatched
|
||||
video on click (see `pins._shelf_card`) — so the tile advertises what is waiting there instead of a
|
||||
runtime it does not have."""
|
||||
card = _base(
|
||||
source="youtube",
|
||||
kind="channel",
|
||||
id=ch.id,
|
||||
title=ch.title,
|
||||
aspect="wide",
|
||||
thumb=ch.thumbnail_url,
|
||||
)
|
||||
card["unwatched_count"] = unwatched
|
||||
return card
|
||||
|
||||
|
||||
def from_playlist(pl: Playlist, item_count: int, thumb: str | None) -> dict:
|
||||
"""A pinned playlist. `thumb` is its first item's thumbnail — a shelf of blank tiles would defeat
|
||||
the point of a surface you are supposed to be able to scan at a glance."""
|
||||
card = _base(
|
||||
source="youtube",
|
||||
kind="playlist",
|
||||
id=str(pl.id),
|
||||
title=pl.name,
|
||||
aspect="wide",
|
||||
thumb=thumb,
|
||||
)
|
||||
card["item_count"] = item_count
|
||||
return card
|
||||
@@ -0,0 +1,104 @@
|
||||
"""The hero: the single suggestion the Home surface leads with.
|
||||
|
||||
One rule, in one function, because the page's entire promise is that it answers rather than offers.
|
||||
Three tiers, tried in order:
|
||||
|
||||
1. **resume** — something already started whose REMAINING time fits the budget. Almost always the
|
||||
right answer: it is the thing the user themselves chose most recently, and finishing it costs no
|
||||
decision at all.
|
||||
2. **comfort** — a pinned shelf entry that fits. The user's own declared fallback, which beats
|
||||
anything we could infer.
|
||||
3. **surprise** — a seeded draw from what is unseen and fits.
|
||||
|
||||
That order is the PAGE's answer. "Mást kérek" rotates it (`start`), so each press offers a different
|
||||
KIND of answer rather than the next row of the same list — the user's call, 2026-09-06, after the
|
||||
button was measured cycling nine resume cards and never once reaching the shelf.
|
||||
|
||||
The tier travels with the card so the client can label it honestly ("Ott hagytad abba" vs "A
|
||||
polcodról" vs "Meglepetés") instead of guessing from the shape of the payload — a kicker that says
|
||||
"where you left off" over a film the user has never opened is exactly the kind of quiet lie the house
|
||||
rules call a dishonest state.
|
||||
|
||||
Budgets are matched against REMAINING time throughout (`budget.remaining_seconds`, which equals the
|
||||
runtime for anything not yet started): with eight minutes left, a 90-minute film is a ten-minute
|
||||
answer.
|
||||
"""
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.discover import cards, pins, rails
|
||||
from app.discover.budget import fits
|
||||
from app.models import User
|
||||
|
||||
HERO_TIERS = ("resume", "comfort", "surprise")
|
||||
|
||||
# How deep the surprise tier draws before giving up. Deliberately deeper than one, so an excluded
|
||||
# pick moves the tier along instead of emptying it (the shuffle is seeded — the same seed always
|
||||
# hands back the same order, so without the headroom "already shown" would mean "nothing left").
|
||||
SURPRISE_DRAW = 12
|
||||
|
||||
|
||||
def pick(
|
||||
db: Session,
|
||||
user: User,
|
||||
budget: str,
|
||||
seed: int,
|
||||
*,
|
||||
resume: list[dict] | None = None,
|
||||
shelf: list[dict] | None = None,
|
||||
exclude: set[str] | None = None,
|
||||
start: int = 0,
|
||||
) -> dict | None:
|
||||
"""`{"tier": ..., "card": ...}`, or None when nothing at all fits.
|
||||
|
||||
`resume` and `shelf` may be passed pre-computed so the consolidated `/api/discover` response does
|
||||
not build the same two rails twice; the reroll endpoint passes neither and pays only for what it
|
||||
reaches.
|
||||
|
||||
`exclude` is what makes "Mást kérek" mean something. A new seed alone only re-draws the SURPRISE
|
||||
tier: the resume tier deterministically returns the most recently progressed thing that fits, so
|
||||
pressing reroll on a user who has something half-finished handed back the identical card and the
|
||||
button looked dead (measured in the browser, 2026-09-06). Skipping the cards already shown makes
|
||||
every press produce a different answer, at every tier.
|
||||
|
||||
`start` ROTATES which tier is tried first — 0 keeps the priority order (the page's own first
|
||||
answer), 1 leads with comfort, 2 with surprise. Exclusion alone was not enough: the resume rail
|
||||
holds up to 20 rows, so the button walked down it and never reached the shelf or a surprise at
|
||||
all (measured, 2026-09-06: twelve presses returned nine cards, every one of them `resume`). The
|
||||
fallthrough is kept, so a press does not come back empty just because the tier it led with has
|
||||
nothing to offer."""
|
||||
skip = exclude or set()
|
||||
|
||||
def _resume() -> dict | None:
|
||||
for card in rails.continue_watching(db, user) if resume is None else resume:
|
||||
playable = rails.playable(db, user.id, card)
|
||||
# Excluded on the card the USER saw (the show), not only on what it resolves to, so a
|
||||
# rerolled-past show does not come straight back as its own next episode.
|
||||
if cards.key(card) in skip or (playable is not None and cards.key(playable) in skip):
|
||||
continue
|
||||
if playable is not None and fits(playable["remaining_seconds"], budget):
|
||||
return playable
|
||||
return None
|
||||
|
||||
def _comfort() -> dict | None:
|
||||
for card in pins.playables(pins.shelf(db, user) if shelf is None else shelf):
|
||||
if cards.key(card) not in skip and fits(card["remaining_seconds"], budget):
|
||||
return card
|
||||
return None
|
||||
|
||||
def _surprise() -> dict | None:
|
||||
# A wider draw than the one card we need: the shuffle is seeded, so without the headroom an
|
||||
# excluded pick would leave the tier empty rather than move past it.
|
||||
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW):
|
||||
if cards.key(card) not in skip:
|
||||
return card
|
||||
return None
|
||||
|
||||
tiers = {"resume": _resume, "comfort": _comfort, "surprise": _surprise}
|
||||
offset = start % len(HERO_TIERS)
|
||||
for tier in HERO_TIERS[offset:] + HERO_TIERS[:offset]:
|
||||
card = tiers[tier]()
|
||||
if card is not None:
|
||||
return {"tier": tier, "card": card}
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,173 @@
|
||||
"""The comfort shelf: pin a thing you can always fall back to, and have it be one click from playing.
|
||||
|
||||
Five kinds are pinnable — a video, a YouTube channel, a Plex show, a Plex movie, a playlist — because
|
||||
comfort media is almost always "put on X", a series or a channel, and only sometimes one particular
|
||||
video. `models.ComfortPin` carries that union as five nullable FKs under a `num_nonnulls(...) = 1`
|
||||
check; this module is where the union is spoken about in one vocabulary, so no call site invents a
|
||||
second one.
|
||||
|
||||
Every shelf card carries a resolved `next`: the thing that actually plays when you click the tile —
|
||||
a show's in-progress-or-next episode, a channel's newest unwatched video, a playlist's first
|
||||
unwatched item. It costs a query per non-leaf pin, which is affordable precisely because this shelf is
|
||||
short and hand-curated, and it buys the property the whole surface exists for: one click, no
|
||||
intermediate page, no second decision.
|
||||
|
||||
Domain module, so nothing here raises HTTP. `resolve_ref` returns None for "no such thing, or not
|
||||
yours" and the route turns that into its 404 — core never imports routes.
|
||||
"""
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.discover import cards, queries
|
||||
from app.models import Channel, ComfortPin, PlexItem, PlexShow, Playlist, User, Video
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, show_card
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
# kind -> the ComfortPin column that holds it. The client's vocabulary and the schema's, in one place.
|
||||
PIN_COLUMNS: dict[str, str] = {
|
||||
"video": "video_id",
|
||||
"channel": "channel_id",
|
||||
"plex_item": "plex_item_id",
|
||||
"plex_show": "plex_show_id",
|
||||
"playlist": "playlist_id",
|
||||
}
|
||||
PIN_KINDS = tuple(PIN_COLUMNS)
|
||||
|
||||
|
||||
def resolve_ref(db: Session, user: User, kind: str, ref: str) -> tuple[str, object] | None:
|
||||
"""Translate the client's (kind, ref) into the (column, value) the row stores, or None.
|
||||
|
||||
The client speaks the ids the rest of the API speaks: a video id, a channel id, a Plex
|
||||
`rating_key`, a playlist id. Plex's internal integer keys never leave the backend, so the
|
||||
translation belongs here rather than in five call sites.
|
||||
|
||||
The COLUMN half of every answer comes from `PIN_COLUMNS`; only the LOOKUP differs per kind
|
||||
(a direct `db.get`, a rating-key translation, an int parse plus an ownership check). Before the
|
||||
E24 review the branches spelled the column names out again and `PIN_COLUMNS` had no readers at
|
||||
all — a constant that called itself the one vocabulary while nothing spoke it."""
|
||||
col = PIN_COLUMNS.get(kind)
|
||||
if col is None:
|
||||
return None
|
||||
if kind == "video":
|
||||
return (col, ref) if db.get(Video, ref) is not None else None
|
||||
if kind == "channel":
|
||||
return (col, ref) if db.get(Channel, ref) is not None else None
|
||||
if kind == "plex_item":
|
||||
it = by_rating_key(db, PlexItem, ref)
|
||||
return (col, it.id) if it is not None else None
|
||||
if kind == "plex_show":
|
||||
sh = by_rating_key(db, PlexShow, ref)
|
||||
return (col, sh.id) if sh is not None else None
|
||||
try:
|
||||
pid = int(ref)
|
||||
except ValueError:
|
||||
return None
|
||||
pl = db.get(Playlist, pid)
|
||||
# A playlist is per-user, unlike the other four (which are shared catalog rows whose PRIVACY
|
||||
# lives in the state tables). Someone else's list must read as "no such thing", not as a 403 —
|
||||
# the same reason `owned_or_404` 404s rather than 403s.
|
||||
return (col, pl.id) if pl is not None and pl.user_id == user.id else None
|
||||
|
||||
|
||||
def list_pins(db: Session, user: User) -> list[ComfortPin]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(ComfortPin)
|
||||
.where(ComfortPin.user_id == user.id)
|
||||
.order_by(ComfortPin.position.asc(), ComfortPin.id.asc())
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def next_position(db: Session, user: User) -> int:
|
||||
"""New pins land at the end of the shelf.
|
||||
|
||||
`max(position) + 1`, not the row count. Positions are never renumbered on removal — `remove_pin`
|
||||
deletes the row, and every FK cascade silently removes a pin when the thing it points at goes
|
||||
away — so past the first hole a count under-counts and would drop the new pin into the MIDDLE of
|
||||
the shelf, or onto a position another pin already holds (nothing in the schema forbids that).
|
||||
Same idiom as `routes/playlists._next_playlist_position`."""
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.max(ComfortPin.position), -1)).where(
|
||||
ComfortPin.user_id == user.id
|
||||
)
|
||||
)
|
||||
+ 1
|
||||
)
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
"""One pin as a unified card with its resolved `next`. None when the pinned row has gone (the FK
|
||||
cascade removes the pin itself, so this is only the window between a delete and a refetch)."""
|
||||
card: dict | None = None
|
||||
nxt: dict | None = None
|
||||
|
||||
if pin.video_id is not None:
|
||||
row = queries.yt_video_row(db, user, pin.video_id)
|
||||
if row is None:
|
||||
return None
|
||||
card = cards.from_video(row)
|
||||
nxt = dict(card) # a video plays itself
|
||||
|
||||
elif pin.channel_id is not None:
|
||||
ch = db.get(Channel, pin.channel_id)
|
||||
if ch is None:
|
||||
return None
|
||||
card = cards.from_channel(ch, queries.channel_unwatched_count(db, user, ch.id))
|
||||
row = queries.channel_next_video(db, user, ch.id)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
elif pin.plex_item_id is not None:
|
||||
it = db.get(PlexItem, pin.plex_item_id)
|
||||
if it is None:
|
||||
return None
|
||||
st = rollup.item_state_map(db, user.id, [it.id]).get(it.id)
|
||||
card = cards.from_plex(leaf_card(it, st))
|
||||
nxt = dict(card) # a movie or a single episode plays itself
|
||||
|
||||
elif pin.plex_show_id is not None:
|
||||
sh = db.get(PlexShow, pin.plex_show_id)
|
||||
if sh is None:
|
||||
return None
|
||||
status = rollup.state_map(db, user.id, [sh.id]).get(sh.id, "new")
|
||||
card = cards.from_plex(show_card(sh, status))
|
||||
ep = queries.next_episode(db, user.id, sh.id)
|
||||
nxt = cards.from_plex(ep) if ep else None
|
||||
|
||||
elif pin.playlist_id is not None:
|
||||
pl = db.get(Playlist, pin.playlist_id)
|
||||
if pl is None:
|
||||
return None
|
||||
count, row = queries.playlist_summary(db, user, pl.id)
|
||||
card = cards.from_playlist(pl, count, row.get("thumbnail_url") if row else None)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
if card is None:
|
||||
return None
|
||||
# A COPY, never the card itself: a self-playing pin (a video, a movie) would otherwise make
|
||||
# `card["next"] is card`, which serializes as a circular reference and 500s the whole shelf. The
|
||||
# copy is taken before `pin_id`/`next` are attached, so `next` stays a plain playable card.
|
||||
card["pin_id"] = pin.id
|
||||
card["next"] = nxt
|
||||
return card
|
||||
|
||||
|
||||
def shelf(db: Session, user: User) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order."""
|
||||
out = []
|
||||
for pin in list_pins(db, user):
|
||||
card = _shelf_card(db, user, pin)
|
||||
if card is not None:
|
||||
out.append(card)
|
||||
return out
|
||||
|
||||
|
||||
def playables(shelf_cards: list[dict]) -> list[dict]:
|
||||
"""The `next` of every shelf entry that has one — what the hero may draw from. A pinned channel
|
||||
with nothing unwatched left, or an empty playlist, simply is not an answer right now."""
|
||||
return [c["next"] for c in shelf_cards if c.get("next")]
|
||||
@@ -0,0 +1,348 @@
|
||||
"""The row fetchers behind the Home surface — one place where "what is startable right now" is
|
||||
expressed as SQL, for both sources.
|
||||
|
||||
Two rules shape this module:
|
||||
|
||||
**Reuse the query layer, not the route.** The YouTube half calls `feedquery.filtered_query` /
|
||||
`sort_keys`, so the Home surface inherits the feed's own definitions of subscribed, blocked,
|
||||
explore-visible and in-progress for free, and cannot drift from them. The Plex half calls
|
||||
`plex.rollup` and the `plex.cards` serializers rather than `routes/plex/discover.py`: that route
|
||||
narrows EVERY rail by the Plex sidebar's `LibraryFilters`, which is right for a surface that HAS a
|
||||
sidebar and wrong for a landing page that does not.
|
||||
|
||||
**One seed drives both halves.** The feed orders a shuffle by `md5(id || seed)` so a keyset cursor
|
||||
stays valid across pages; the Plex surprise rail uses `ORDER BY random()`, which is fine for a single
|
||||
un-paged draw but cannot be reproduced. A cross-source reroll needs both halves to move together —
|
||||
otherwise "Mást kérek" re-draws the Plex picks while the YouTube ones stand still — so the Plex side
|
||||
here takes the feed's seeded-md5 idiom instead of `random()`.
|
||||
"""
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.discover.budget import band
|
||||
from app.feedquery import (
|
||||
feed_columns,
|
||||
filtered_query,
|
||||
not_blocked_cond,
|
||||
order_by,
|
||||
serialize,
|
||||
sort_keys,
|
||||
)
|
||||
from app.models import (
|
||||
Channel,
|
||||
PlaylistItem,
|
||||
PlexItem,
|
||||
PlexShow,
|
||||
PlexState,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, movie_card, show_card
|
||||
from app.plex.query import apply_show_state_filter, content_libs
|
||||
|
||||
# How many rows a single rail may carry. Matches `routes/plex/discover.py`'s RAIL_LIMIT on purpose —
|
||||
# the two surfaces should feel like the same app, and a rail is a rail.
|
||||
RAIL_LIMIT = 20
|
||||
|
||||
# The feed parameters a Home rail runs with. Spelled out rather than borrowed from the route's
|
||||
# `_feed_params` defaults, because those are an HTTP contract that may change for the feed's own
|
||||
# reasons; the landing page's scope should not move with them silently.
|
||||
_FEED_BASE: dict = {
|
||||
"tags": [],
|
||||
"tag_modes": {},
|
||||
"untagged": False,
|
||||
"channel_ids": [],
|
||||
"q": None,
|
||||
"min_duration": None,
|
||||
"max_duration": None,
|
||||
"max_age_days": None,
|
||||
"published_after": None,
|
||||
"published_before": None,
|
||||
"show_normal": True,
|
||||
"include_shorts": False,
|
||||
"include_live": False,
|
||||
"show": "unwatched",
|
||||
"scope": "my",
|
||||
"library_source": "organic",
|
||||
}
|
||||
|
||||
|
||||
def _seeded(col, seed: int):
|
||||
"""The feed's deterministic shuffle expression, reused for Plex so one seed moves both halves."""
|
||||
return func.md5(func.concat(col, str(seed)))
|
||||
|
||||
|
||||
# ── YouTube ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def yt_in_progress(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[tuple[object, dict]]:
|
||||
"""Started-but-not-finished videos, most recently progressed first, as (progressed_at, row).
|
||||
|
||||
The predicate is `feedquery`'s `show="in_progress"` verbatim (a stored resume position, status not
|
||||
watched/hidden) — but the query is built here rather than through `filtered_query` because this
|
||||
rail wants an ordering by `progress_updated_at`, and that column lives on the state alias the feed
|
||||
query keeps private. Driving from `video_states` is also the faster plan: it is a per-user table of
|
||||
a few hundred rows, against a catalog of many thousands.
|
||||
|
||||
Building it here means the feed's OTHER predicates have to be chosen deliberately rather than
|
||||
inherited. `not_blocked_cond` is shared, because "never show me this channel" outranks "you
|
||||
started it" — blocking leaves the state row untouched, so without this the video the user just
|
||||
banned heads the resume rail. The feed's SCOPE clauses (subscribed / via_explore accessibility)
|
||||
are deliberately NOT applied, for the reason `yt_video_row` gives: something you have already
|
||||
started must stay resumable even after you unsubscribe or an exploration expires."""
|
||||
st = aliased(VideoState)
|
||||
q = (
|
||||
select(*feed_columns(user, st), st.progress_updated_at.label("progressed_at"))
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.join(st, and_(st.video_id == Video.id, st.user_id == user.id))
|
||||
.where(
|
||||
st.position_seconds > 0,
|
||||
st.status.notin_(("watched", "hidden")),
|
||||
Video.unavailable_since.is_(None),
|
||||
not_blocked_cond(user),
|
||||
)
|
||||
.order_by(st.progress_updated_at.desc().nullslast())
|
||||
.limit(limit)
|
||||
)
|
||||
return [(r.progressed_at, serialize(r)) for r in db.execute(q).all()]
|
||||
|
||||
|
||||
def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) -> list[dict]:
|
||||
"""Unwatched videos from the user's own feed scope, in seeded-shuffle order, inside the budget.
|
||||
|
||||
Shorts are opened up only for the shortest band: "I have ten minutes" is the one answer a Short is
|
||||
actually a good response to, and the feed's own default keeps them out otherwise."""
|
||||
lo, hi = band(budget)
|
||||
params = dict(_FEED_BASE, min_duration=lo, max_duration=hi, include_shorts=budget == "short")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys("shuffle", seed, rank_expr)
|
||||
# Over-fetch, then drop anything already started. The feed's `unwatched` scope keeps watched and
|
||||
# hidden out but deliberately keeps in-progress rows in (they belong in the feed). Here they must
|
||||
# go: the resume tier has already had its chance at them, and it may have passed precisely because
|
||||
# the REMAINING time did not fit the budget — offering the same item back as a surprise, whose
|
||||
# runtime does fit, would label a half-finished thing "Meglepetés" and contradict the question.
|
||||
rows = db.execute(query.order_by(*order_by(keys)).limit(limit * 4)).all()
|
||||
out = [serialize(r) for r in rows]
|
||||
return [r for r in out if not r["position_seconds"]][:limit]
|
||||
|
||||
|
||||
def channel_next_video(db: Session, user: User, channel_id: str) -> dict | None:
|
||||
"""The newest unwatched video of a pinned channel — what "put on that channel" resolves to."""
|
||||
# `library_source="all"`: a pinned channel means the whole channel, including videos that only
|
||||
# ever arrived through a search. The feed's "organic" default is a feed-scoping choice, not a
|
||||
# statement about what belongs to a channel.
|
||||
params = dict(_FEED_BASE, channel_ids=[channel_id], scope="all", library_source="all")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys("newest", 0, rank_expr)
|
||||
row = db.execute(query.order_by(*order_by(keys)).limit(1)).first()
|
||||
return serialize(row) if row is not None else None
|
||||
|
||||
|
||||
def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||
"""One video as a feed row, with this user's watch state.
|
||||
|
||||
Built directly rather than through `filtered_query` on purpose: a PINNED video must keep working
|
||||
even when it falls outside the feed's scope — the user unsubscribed from the channel, or blocked
|
||||
it, or the video only ever arrived through a search. The shelf is the surface you fall back to;
|
||||
it silently losing an entry because a filter moved is exactly the failure it must not have."""
|
||||
st = aliased(VideoState)
|
||||
row = db.execute(
|
||||
select(*feed_columns(user, st))
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(st, and_(st.video_id == Video.id, st.user_id == user.id))
|
||||
.where(Video.id == video_id)
|
||||
).first()
|
||||
return serialize(row) if row is not None else None
|
||||
|
||||
|
||||
def channel_unwatched_count(db: Session, user: User, channel_id: str) -> int:
|
||||
"""How many unwatched videos a pinned channel is holding — the tile's only honest promise, since a
|
||||
channel has no runtime of its own."""
|
||||
params = dict(_FEED_BASE, channel_ids=[channel_id], scope="all", library_source="all")
|
||||
query, _ = filtered_query(db, user, **params)
|
||||
return int(db.execute(select(func.count()).select_from(query.subquery())).scalar() or 0)
|
||||
|
||||
|
||||
def playlist_summary(db: Session, user: User, playlist_id: int) -> tuple[int, dict | None]:
|
||||
"""(item count, the row to play next) for a pinned playlist — its first not-yet-watched item, or
|
||||
its first item once the whole list is watched. The same row doubles as the tile's artwork, so a
|
||||
shelf of playlists is scannable instead of a row of blank boxes."""
|
||||
st = aliased(VideoState)
|
||||
rows = db.execute(
|
||||
select(*feed_columns(user, st), PlaylistItem.position)
|
||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(st, and_(st.video_id == Video.id, st.user_id == user.id))
|
||||
.where(PlaylistItem.playlist_id == playlist_id)
|
||||
.order_by(PlaylistItem.position.asc())
|
||||
).all()
|
||||
if not rows:
|
||||
return 0, None
|
||||
nxt = next((r for r in rows if (r.status or "new") not in ("watched", "hidden")), rows[0])
|
||||
return len(rows), serialize(nxt)
|
||||
|
||||
|
||||
# ── Plex ──────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _unseen_movie_conds(uid: int):
|
||||
"""A movie the user has neither finished nor hidden. A LEFT-JOIN-IS-NULL would also drop rows that
|
||||
merely carry a resume position, which are exactly the ones Continue watching wants back."""
|
||||
seen = (
|
||||
select(1)
|
||||
.select_from(PlexState)
|
||||
.where(
|
||||
PlexState.item_id == PlexItem.id,
|
||||
PlexState.user_id == uid,
|
||||
or_(
|
||||
PlexState.status.in_(("watched", "hidden")),
|
||||
# Started counts as "not a surprise" too — see the note in `yt_surprise`.
|
||||
PlexState.position_seconds > 0,
|
||||
),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
return ~seen
|
||||
|
||||
|
||||
def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tuple[object, dict]]:
|
||||
"""In-progress movies + partially-watched shows as (progressed_at, card).
|
||||
|
||||
Mirrors `routes/plex/discover.py::_continue_watching` — same `PlexState` predicate for movies (a
|
||||
NULL status still counts as started, so `notin_` alone would drop it), and the same shared
|
||||
`rollup.agg_subquery` + `apply_show_state_filter` for shows, so the two rails cannot disagree on
|
||||
what "in progress" means. What it does NOT inherit is that route's `LibraryFilters` narrowing:
|
||||
this surface has no Plex sidebar to read a filter from."""
|
||||
movie_libs, show_libs = content_libs(db)
|
||||
out: list[tuple[object, dict]] = []
|
||||
|
||||
if movie_libs:
|
||||
mq = (
|
||||
select(PlexItem, PlexState)
|
||||
.join(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
|
||||
.where(
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.library_id.in_(movie_libs),
|
||||
PlexState.position_seconds > 0,
|
||||
or_(
|
||||
PlexState.status.is_(None),
|
||||
PlexState.status.notin_(("watched", "hidden")),
|
||||
),
|
||||
)
|
||||
.order_by(PlexState.progress_updated_at.desc().nullslast())
|
||||
.limit(limit)
|
||||
)
|
||||
for it, st in db.execute(mq).all():
|
||||
out.append((st.progress_updated_at, movie_card(it, st)))
|
||||
|
||||
if show_libs:
|
||||
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
||||
sq = apply_show_state_filter(
|
||||
select(PlexShow, prog.c.resumed_at).join(prog, prog.c.show_id == PlexShow.id),
|
||||
prog.c.total,
|
||||
prog.c.watched,
|
||||
prog.c.inprog,
|
||||
"in_progress",
|
||||
).where(PlexShow.library_id.in_(show_libs))
|
||||
for sh, resumed_at in db.execute(
|
||||
sq.order_by(prog.c.resumed_at.desc().nullslast()).limit(limit)
|
||||
).all():
|
||||
out.append((resumed_at, show_card(sh, "in_progress")))
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) -> list[dict]:
|
||||
"""Unseen playable movies inside the budget, in seeded-shuffle order.
|
||||
|
||||
Movies only. A show is not a thing you can start playing, and resolving every candidate show to
|
||||
its next episode just to draw three cards would be a query per row for a rail whose whole value is
|
||||
that it answers instantly — shows reach this surface through Continue watching and the comfort
|
||||
shelf, which is where "put the next episode on" actually belongs.
|
||||
|
||||
No `playable` filter, matching the Plex surprise rail: `PlexItem.playable` is a playback CLASS
|
||||
("direct" / "remux" / "transcode"), not a yes/no, and NULL there means "not probed yet" rather than
|
||||
"will not play". Excluding un-probed items would quietly hide a fresh library from the one surface
|
||||
that is supposed to have an answer. The class travels on the card, so the client still shows it."""
|
||||
movie_libs, _ = content_libs(db)
|
||||
if not movie_libs:
|
||||
return []
|
||||
lo, hi = band(budget)
|
||||
q = (
|
||||
select(PlexItem)
|
||||
.where(
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.library_id.in_(movie_libs),
|
||||
_unseen_movie_conds(uid),
|
||||
)
|
||||
.order_by(_seeded(PlexItem.rating_key, seed))
|
||||
.limit(limit)
|
||||
)
|
||||
if lo is not None:
|
||||
q = q.where(PlexItem.duration_s >= lo)
|
||||
if hi is not None:
|
||||
q = q.where(PlexItem.duration_s <= hi)
|
||||
items = db.execute(q).scalars().all()
|
||||
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||
return [movie_card(it, states.get(it.id)) for it in items]
|
||||
|
||||
|
||||
# Watch states that take an episode out of the running for "what would play next".
|
||||
_DONE = ("watched", "hidden")
|
||||
|
||||
|
||||
def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
||||
"""What "put that show on" means: the episode in progress, else the first unwatched one, else the
|
||||
first — over the same `(season, episode)` broadcast ordering `detail.py::_episode_nav` uses.
|
||||
|
||||
Three ordered `LIMIT 1` probes rather than one scan picked apart in Python. The Python version
|
||||
loaded EVERY episode of the show to choose one of them — 236 rows for Friends, 227 for JAG — which
|
||||
the comfort shelf could just about afford (a handful of hand-picked pins) and the resume rail
|
||||
could not, since it resolves a show per rail row on the landing page.
|
||||
|
||||
Returned as a `leaf_card` so it carries the show title the episode row itself does not."""
|
||||
asc = (
|
||||
PlexItem.season_number.asc().nullsfirst(),
|
||||
PlexItem.episode_number.asc().nullsfirst(),
|
||||
)
|
||||
base = (
|
||||
select(PlexItem, PlexState)
|
||||
.outerjoin(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
|
||||
.where(PlexItem.show_id == show_id, PlexItem.kind == "episode")
|
||||
.limit(1)
|
||||
)
|
||||
# In progress — the LAST one in broadcast order, which is `plex.rollup.rollup`'s `inprog[-1]`,
|
||||
# i.e. what the show page's own Resume button plays. Two episodes of one show are in progress at
|
||||
# once more often than it sounds (`watch_sync._reconcile_inprogress_movies` documents that a
|
||||
# scrub to the end leaves a stale `status='new', position>0` row until the daily reconcile), and
|
||||
# taking opposite ends of that set is how one show came to have two different "continue" answers:
|
||||
# the Home tile ranked it by its LATEST touch and then offered the EARLIEST started episode.
|
||||
# `position_seconds > 0` already requires the state row to exist, so no null guard.
|
||||
pick = db.execute(
|
||||
base.where(PlexState.position_seconds > 0, PlexState.status.notin_(_DONE)).order_by(
|
||||
PlexItem.season_number.desc().nullslast(),
|
||||
PlexItem.episode_number.desc().nullslast(),
|
||||
)
|
||||
).first()
|
||||
# Unwatched — INCLUDING an episode with no state row at all, which is the common case and the one
|
||||
# a bare `status NOT IN (...)` silently drops (NULL is not false, it is unknown, so the row fails
|
||||
# the filter).
|
||||
pick = (
|
||||
pick
|
||||
or db.execute(
|
||||
base.where(or_(PlexState.id.is_(None), PlexState.status.notin_(_DONE))).order_by(*asc)
|
||||
).first()
|
||||
)
|
||||
# Fully watched show: start it again from the top.
|
||||
pick = pick or db.execute(base.order_by(*asc)).first()
|
||||
if pick is None:
|
||||
return None
|
||||
|
||||
ep, st = pick
|
||||
show = db.get(PlexShow, show_id)
|
||||
return leaf_card(
|
||||
ep, st, show_title=show.title if show else None, show_rk=show.rating_key if show else None
|
||||
)
|
||||
@@ -0,0 +1,97 @@
|
||||
"""The Home surface's rails: cross-source lists, assembled from `queries.py` and emitted as unified
|
||||
cards.
|
||||
|
||||
The whole point of this module is that a rail is ONE list, not a YouTube list beside a Plex list. Two
|
||||
side-by-side rails would just move the choice the user could not make ("which half do I look at?") one
|
||||
level up, which is the opposite of what this epic is for.
|
||||
"""
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.discover import cards, queries
|
||||
from app.discover.queries import RAIL_LIMIT
|
||||
from app.models import PlexShow, User
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.plex.rollup import NO_TS
|
||||
|
||||
|
||||
def plex_available(db: Session) -> bool:
|
||||
"""Plex is a deployment-level toggle, not a per-user one (see `routes/me.py`). With it off, the
|
||||
Home rails are YouTube-only and say nothing about Plex."""
|
||||
return sysconfig.get_bool(db, "plex_enabled")
|
||||
|
||||
|
||||
def playable(db: Session, uid: int, card: dict) -> dict | None:
|
||||
"""The thing that would actually START from this card — itself for anything with a runtime, and
|
||||
for a Plex SHOW the episode that is next (in progress, else the first unwatched, else the first).
|
||||
|
||||
ONE function, because three surfaces need the same answer and a second implementation is how the
|
||||
shelf and the rail came to disagree in the first place: the hero calls it to decide what it is
|
||||
offering, `continue_watching` calls it to fill each show tile's `next`, and the comfort shelf
|
||||
resolves the same way in `pins`. None when the show has no episodes mirrored yet.
|
||||
|
||||
A card that already carries a resolved `next` (a rail row) is taken at its word rather than
|
||||
re-queried — the hero reads the rail it was handed."""
|
||||
if not (card.get("source") == "plex" and card.get("kind") == "show"):
|
||||
return card
|
||||
if "next" in card:
|
||||
return card["next"]
|
||||
sh = by_rating_key(db, PlexShow, card["id"])
|
||||
if sh is None:
|
||||
return None
|
||||
ep = queries.next_episode(db, uid, sh.id)
|
||||
return cards.from_plex(ep) if ep else None
|
||||
|
||||
|
||||
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[dict]:
|
||||
"""One resume rail across both sources, most recently progressed first.
|
||||
|
||||
A Plex SHOW enters as a show CARD — that is what the Plex rail shows, and it is the artwork the
|
||||
user recognises — but it carries `next`: the episode that would actually start. Without it a
|
||||
click on a half-watched series landed on the show page with six seasons to choose between, which
|
||||
is the decision this whole surface exists to remove (the comfort shelf and the hero already
|
||||
resolved; only this rail did not). The client plays `next ?? card`, so the tile stays a poster and
|
||||
the click stays one click."""
|
||||
rows: list[tuple[object, dict]] = [
|
||||
(ts, cards.from_video(r)) for ts, r in queries.yt_in_progress(db, user, limit)
|
||||
]
|
||||
if plex_available(db):
|
||||
rows += [(ts, cards.from_plex(c)) for ts, c in queries.plex_in_progress(db, user.id, limit)]
|
||||
rows.sort(key=lambda r: r[0] or NO_TS, reverse=True)
|
||||
out = [card for _, card in rows[:limit]]
|
||||
# Resolve AFTER the slice: only the rows that survive it are worth a query.
|
||||
for card in out:
|
||||
if card["kind"] == "show":
|
||||
card["next"] = playable(db, user.id, card)
|
||||
return out
|
||||
|
||||
|
||||
def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) -> list[dict]:
|
||||
"""The hero's draw pool: unseen things that fit the budget, from both sources under ONE seed.
|
||||
|
||||
Interleaved rather than concatenated — the same alternation `routes/plex/discover.py` uses for its
|
||||
movie/show split — so a short pool never comes back all-YouTube or all-Plex just because one side
|
||||
happened to be queried first.
|
||||
|
||||
WHICH source leads is decided by the seed, not by a hardcoded order. The hero draws a pool of ONE,
|
||||
so a fixed order would mean the Plex library could never answer "egy este" while a single YouTube
|
||||
video also fit — the reroll would look broken and the cross-source promise would quietly be a
|
||||
YouTube promise. Parity on the same seed that orders the rows keeps it deterministic: the same
|
||||
seed always gives the same answer, and pressing "Mást kérek" moves both halves together."""
|
||||
halves = [
|
||||
(cards.from_video, queries.yt_surprise(db, user, budget, seed, limit)),
|
||||
(
|
||||
cards.from_plex,
|
||||
queries.plex_surprise(db, user.id, budget, seed, limit) if plex_available(db) else [],
|
||||
),
|
||||
]
|
||||
if seed % 2:
|
||||
halves.reverse()
|
||||
|
||||
out: list[dict] = []
|
||||
for i in range(max(len(rows) for _, rows in halves)):
|
||||
for to_card, rows in halves:
|
||||
if i < len(rows):
|
||||
out.append(to_card(rows[i]))
|
||||
return out[:limit]
|
||||
@@ -11,10 +11,22 @@ Resolution works both before extraction (from source_kind / URL host) and after
|
||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||
"""
|
||||
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||||
|
||||
from app.config import settings
|
||||
|
||||
# Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry
|
||||
# below and the failure catalog (errors.py) decide on it, so the wording exists once.
|
||||
FACEBOOK_PARSE_WALL = "cannot parse data"
|
||||
# A source's own verdicts on a link, raised from `resolve_url`. errors.py classifies them; it imports
|
||||
# them from here because this module must not import errors (errors → credentials → adapters).
|
||||
POST_WITHOUT_VIDEO_MESSAGE = "No single video was found in this post."
|
||||
POST_NOT_SHOWN_MESSAGE = "Facebook did not show this post to the signed-in session."
|
||||
|
||||
|
||||
class SourceAdapter:
|
||||
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
|
||||
@@ -22,9 +34,21 @@ class SourceAdapter:
|
||||
key = "generic"
|
||||
|
||||
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||||
def client_sets(self) -> list:
|
||||
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
|
||||
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
"""The link to hand yt-dlp for `url`, resolved BEFORE the first attempt. `ydl_opts` are the
|
||||
download's own options (cookie, timeouts, address family) for any page this needs to read.
|
||||
|
||||
Raises a RuntimeError carrying one of this module's verdicts when the page read shows the
|
||||
download cannot go ahead: the content is there but holds no single video
|
||||
(`POST_WITHOUT_VIDEO_MESSAGE`), or the page is not the content at all — the login page a
|
||||
refused cookie gets (`POST_NOT_SHOWN_MESSAGE`), never the no-video verdict. A failed page read
|
||||
is raised as itself, so it is classified like any other download error. Generic: `url`."""
|
||||
return url
|
||||
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
"""Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one
|
||||
is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`).
|
||||
`authed` is whether the acting user's cookie file goes with the download."""
|
||||
return [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
@@ -37,6 +61,10 @@ class SourceAdapter:
|
||||
return False
|
||||
|
||||
# --- metadata -----------------------------------------------------------------------------
|
||||
def title(self, info: dict) -> str | None:
|
||||
"""The download's raw title, before `titles.normalize_title` cleans it. Generic: yt-dlp's."""
|
||||
return info.get("title")
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||||
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||||
@@ -65,12 +93,46 @@ class YouTubeAdapter(SourceAdapter):
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
# Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then
|
||||
# refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves
|
||||
# between the player response and the stream, so this is the same per-client flakiness one
|
||||
# step later — and it used to end the job on attempt 1, because the text classifies as
|
||||
# `network` and nothing here matched it. The format URLs belong to the client that minted
|
||||
# them, so the next client set is a real second chance.
|
||||
"unable to download video data",
|
||||
)
|
||||
# Measured in the worker against a REAL account cookie (2026-09-16, yt-dlp 2026.07.04): with
|
||||
# `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
|
||||
# cookies`, drops it, and the request that is left with no client fails with "Requested format is
|
||||
# not available" — which is already a retriable marker above, so the cost is a wasted attempt
|
||||
# rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every
|
||||
# signed-in download would burn its first attempt that way before reaching the POT-backed web
|
||||
# fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
|
||||
# ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
|
||||
# that moves the table turns the gate red instead of silently un-authing every YouTube download.
|
||||
_COOKIE_CAPABLE = frozenset(
|
||||
{
|
||||
"mweb",
|
||||
"tv",
|
||||
"tv_downgraded",
|
||||
"web",
|
||||
"web_creator",
|
||||
"web_embedded",
|
||||
"web_music",
|
||||
"web_safari",
|
||||
}
|
||||
)
|
||||
# Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all.
|
||||
# Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for
|
||||
# the per-client flakiness that the two-tier ladder gives a signed-out download.
|
||||
_AUTHED_DEFAULT = ("web_safari", "web")
|
||||
|
||||
def client_sets(self) -> list:
|
||||
return [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c]
|
||||
if authed:
|
||||
kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets]
|
||||
return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)]
|
||||
return sets or [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||||
@@ -93,6 +155,63 @@ class YouTubeAdapter(SourceAdapter):
|
||||
|
||||
class FacebookAdapter(SourceAdapter):
|
||||
key = "facebook"
|
||||
# Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||||
# with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a
|
||||
# SECOND attempt behind that one error, not the default: the links that already download with the
|
||||
# cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without
|
||||
# a cookie the same error is the login wall, which no disguise gets past.
|
||||
_IMPERSONATE = "impersonate"
|
||||
# A post page is ~4–5 MB of inline JSON; anything far past that is not a post page.
|
||||
_POST_PAGE_MAX_BYTES = 16 * 1024 * 1024
|
||||
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
# Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
|
||||
# downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
|
||||
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
|
||||
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
|
||||
post_id = facebook_group_post_id(url)
|
||||
if post_id is None or not ydl_opts.get("cookiefile"):
|
||||
return url
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
|
||||
page = raw.decode("utf-8", "replace")
|
||||
# Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
|
||||
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
|
||||
if _quoted(post_id) not in page:
|
||||
raise RuntimeError(POST_NOT_SHOWN_MESSAGE)
|
||||
video_id = facebook_post_video_id(page, post_id)
|
||||
if video_id is None:
|
||||
# Never fall back to the post link: that is exactly where the wrong video comes from.
|
||||
raise RuntimeError(POST_WITHOUT_VIDEO_MESSAGE)
|
||||
return f"https://www.facebook.com/watch/?v={video_id}"
|
||||
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
return [None, self._IMPERSONATE] if authed else [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
if clients == self._IMPERSONATE:
|
||||
return {"impersonate": ImpersonateTarget.from_str("chrome")}
|
||||
return {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
return FACEBOOK_PARSE_WALL in error_text.lower()
|
||||
|
||||
# Measured (2026-09-14): a SIGNED-IN read gives the placeholder "Video" (plain and as Chrome), and an
|
||||
# 18+ reel only readable as Chrome gives "Facebook"; the post's text is still in `description`. A
|
||||
# signed-out read titles the page with that same text behind the engagement counts, cut short.
|
||||
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
|
||||
|
||||
def title(self, info: dict) -> str | None:
|
||||
raw = info.get("title")
|
||||
if (raw or "").strip().lower() not in self._PLACEHOLDER_TITLES:
|
||||
return raw
|
||||
line = next(
|
||||
(ln.strip() for ln in (info.get("description") or "").splitlines() if ln.strip()), ""
|
||||
)
|
||||
# The whole first line, never cut here with an ellipsis: normalize_title's NFKC turns "…" into
|
||||
# "..." and then collapses it to ".", which read as a finished sentence ("… live at."). The
|
||||
# stored title is unbounded; display_name and the file name clip it themselves.
|
||||
return line or raw # no description either (the 18+ reel): nothing better on the page
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||||
@@ -107,6 +226,39 @@ class FacebookAdapter(SourceAdapter):
|
||||
return True
|
||||
|
||||
|
||||
_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
|
||||
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
|
||||
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')
|
||||
|
||||
|
||||
def _quoted(post_id: str) -> str:
|
||||
# How the page JSON carries the id: quoted, so a longer id that merely starts with it never counts.
|
||||
return f'"{post_id}"'
|
||||
|
||||
|
||||
def facebook_group_post_id(url: str) -> str | None:
|
||||
"""The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None."""
|
||||
parsed = urlparse(url)
|
||||
if not _host_is(parsed.netloc.lower(), "facebook.com"):
|
||||
return None
|
||||
match = _FB_GROUP_POST_PATH.match(parsed.path)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def facebook_post_video_id(page: str, post_id: str) -> str | None:
|
||||
"""The id of the one video attached to `post_id`, read from its page: the Video nodes of the script
|
||||
blocks that carry the post id (the page's other ~10 videos — feed, reels — sit in other blocks).
|
||||
None for no video AND for more than one: guessing between two would be the wrong-video bug again."""
|
||||
quoted = _quoted(post_id)
|
||||
ids = {
|
||||
vid
|
||||
for block in _SCRIPT_BLOCK.findall(page)
|
||||
if quoted in block
|
||||
for vid in _VIDEO_NODE.findall(block)
|
||||
}
|
||||
return ids.pop() if len(ids) == 1 else None
|
||||
|
||||
|
||||
class RedditAdapter(SourceAdapter):
|
||||
key = "reddit"
|
||||
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
"""Per-user, per-host download cookies — the single source of truth for "does the acting user's
|
||||
cookie apply to this download, and how does that keep the result private".
|
||||
|
||||
Two call sites depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
Two sides depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
in prose):
|
||||
* `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user.
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user. Retry
|
||||
(`service.resume`) and Re-download (`service.redownload`) ask the same question again, so a
|
||||
cookie the user added after a failure goes with the next attempt.
|
||||
* the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it
|
||||
is downloading into is that private, salted identity — so a cookie can never poison a shared
|
||||
asset (the load-bearing cross-user privacy fix).
|
||||
|
||||
Both go through `cookie_salt` / `authed_format_sig` / `host_key_for_url` here, so they can't drift.
|
||||
Both key the private identity on `cookie_salt` / `host_key_for_url` here, so they can't drift.
|
||||
|
||||
Security controls enforced here:
|
||||
* **host scope (SEC-12)** — `host_key_for_url` matches the download URL's netloc against a curated
|
||||
@@ -30,7 +32,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential
|
||||
from app.models import DownloadCredential, MediaAsset
|
||||
from app.security import decrypt_optional, encrypt
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.credentials")
|
||||
@@ -53,6 +55,11 @@ class HostSpec:
|
||||
# all derive from this, so there is nothing else to wire up.
|
||||
SUPPORTED_HOSTS: tuple[HostSpec, ...] = (
|
||||
HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")),
|
||||
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
||||
# job 273: every player client fails on that one video while its neighbours download fine, and
|
||||
# the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a
|
||||
# row here the `login_required` copy sent the user to a dropdown that could not help them.
|
||||
HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")),
|
||||
HostSpec("instagram", "Instagram", ("instagram.com",)),
|
||||
HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")),
|
||||
HostSpec("tiktok", "TikTok", ("tiktok.com",)),
|
||||
@@ -99,11 +106,38 @@ def cookie_salt(user_id: int, host_key: str) -> str:
|
||||
|
||||
|
||||
def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str:
|
||||
"""The salted `format_sig` an authed download for (user, host) resolves to — the private asset
|
||||
identity. Both enqueue (to create it) and the worker (to recognise it) compute it HERE."""
|
||||
"""The salted `format_sig` an authed download for (user, host) resolves to under the current
|
||||
signature version — the private asset enqueue creates, and one of the identities `is_authed_asset`
|
||||
recognises. Neither calls it; the tests use it to hold the two to each other."""
|
||||
return formats.format_sig(spec, salt=cookie_salt(user_id, host_key))
|
||||
|
||||
|
||||
def rebuild_salt(asset_id: int, cookie: str | None) -> str:
|
||||
"""The salt of a Re-download's fresh asset: its OWN row id (so every rebuild is a new identity no
|
||||
enqueue can ever compute), prefixed with the user's `cookie_salt` when a cookie applies. The id is
|
||||
what makes it recognisable — the worker has the asset row, so `is_authed_asset` can recompute this
|
||||
one, where a salt built from the job's previous asset id could not be."""
|
||||
return f"{cookie}:rebuild:{asset_id}" if cookie else f"rebuild:{asset_id}"
|
||||
|
||||
|
||||
def is_authed_asset(asset: MediaAsset, spec: dict, user_id: int, url: str | None) -> bool:
|
||||
"""Whether an asset is `user_id`'s private, cookie-salted identity for `url`'s host — the one
|
||||
kind of asset a cookie is ever attached to: the one enqueue creates (`authed_format_sig`), or a
|
||||
Re-download's rebuild of it (`rebuild_salt`). The worker's cookie gate asks it before a download,
|
||||
and the failure diagnosis asks it after one (a failure on this asset happened WITH the cookie).
|
||||
Either signature version counts (`formats.format_sigs`): a job queued before a deploy that bumped
|
||||
the version still downloads into the asset it was given."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return False
|
||||
salt = cookie_salt(user_id, host_key)
|
||||
if asset.format_sig in formats.format_sigs(spec, salt=salt):
|
||||
return True
|
||||
return asset.id is not None and asset.format_sig in formats.format_sigs(
|
||||
spec, salt=rebuild_salt(asset.id, salt)
|
||||
)
|
||||
|
||||
|
||||
def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
||||
"""At enqueue: the cache salt to apply if this user has a cookie for `url`'s host, else None (a
|
||||
normal shared download). Returns the salt string so the caller passes it straight to
|
||||
@@ -199,20 +233,19 @@ def status_list(db: Session, user_id: int) -> list[dict]:
|
||||
# --- worker: resolve + write the cookie file --------------------------------------------------
|
||||
|
||||
|
||||
def resolve_cookiefile(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
url: str | None,
|
||||
asset_format_sig: str,
|
||||
spec: dict,
|
||||
staging: Path,
|
||||
) -> Path | None:
|
||||
"""In the worker child: the cookies.txt to hand yt-dlp for this download, or None.
|
||||
def usable_cookie(
|
||||
db: Session, user_id: int, url: str | None, asset: MediaAsset, spec: dict
|
||||
) -> str | None:
|
||||
"""The user's cookie text a download of `url` into `asset` goes with, or None — the ONE decision
|
||||
whether a download is signed in. The worker child writes it out for yt-dlp (`resolve_cookiefile`);
|
||||
the worker asks it again for a job that inherits a failed asset. That second ask sees the
|
||||
credential as it is when the job is claimed: a cookie the user removed or added since the failed
|
||||
download changes this job's explanation to fit the cookie its Retry would go with.
|
||||
|
||||
The double gate that prevents cross-user leakage:
|
||||
1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND
|
||||
2. the asset being downloaded into must BE this user's private, salted identity
|
||||
(`asset_format_sig == authed_format_sig(...)`).
|
||||
(`is_authed_asset`).
|
||||
|
||||
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
||||
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
||||
@@ -224,18 +257,31 @@ def resolve_cookiefile(
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
if cred is None:
|
||||
return None
|
||||
if asset_format_sig != authed_format_sig(spec, user_id, host_key):
|
||||
if not is_authed_asset(asset, spec, user_id, url):
|
||||
return None # shared asset — never attach a cookie to a non-private identity
|
||||
raw = decrypt_optional(cred.cookies_enc)
|
||||
if not raw:
|
||||
log.warning(
|
||||
"download cookie for user %s host %s could not be decrypted (rotated key?) — "
|
||||
"proceeding without it",
|
||||
"treated as no cookie",
|
||||
user_id,
|
||||
host_key,
|
||||
)
|
||||
return None
|
||||
return _write_cookiefile(raw, staging)
|
||||
return raw
|
||||
|
||||
|
||||
def resolve_cookiefile(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
url: str | None,
|
||||
asset: MediaAsset,
|
||||
spec: dict,
|
||||
staging: Path,
|
||||
) -> Path | None:
|
||||
"""In the worker child: the cookies.txt to hand yt-dlp for this download (`usable_cookie`), or None."""
|
||||
raw = usable_cookie(db, user_id, url, asset, spec)
|
||||
return _write_cookiefile(raw, staging) if raw else None
|
||||
|
||||
|
||||
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code.
|
||||
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code — and, where a site's
|
||||
failure has been measured, into that site's entry of the failure catalog.
|
||||
|
||||
The worker stores the raw exception text (truncated) for technical detail, but a raw
|
||||
`ERROR: [youtube] <id>: Private video. Sign in…` is noise to a user. `classify()` maps that text
|
||||
@@ -6,21 +7,51 @@ to one of a small set of category codes; the frontend turns the code into a loca
|
||||
explanation (and decides whether a retry makes sense). The raw text stays available behind a
|
||||
"details" disclosure. This is the same string-matching approach the worker already trusts for
|
||||
`_RETRIABLE_MARKERS` — kept here as the single source of truth for "what kind of failure is this".
|
||||
|
||||
**The failure catalog** (`_CATALOG`, E11.5). A category is generic: "Sign-in required" cannot tell a
|
||||
Facebook user that the cookie they already gave was refused, and yt-dlp words several different
|
||||
Facebook walls identically (`Cannot parse data`). A catalog entry is scoped to ONE host and may also
|
||||
depend on whether a cookie was attached and on the kind of link, and names a `detail` code the UI has
|
||||
its own sentence for — why it failed on that site, and what the user can do. `diagnose()` tries the
|
||||
catalog first and falls back to the category. How an entry is added, every time a failure is reported:
|
||||
1. reproduce it on dev (with and without the user's cookie) and keep the verbatim yt-dlp text;
|
||||
2. add the entry below, scoped as narrowly as the measurement allows (a marker that is yt-dlp's
|
||||
generic wording for every site must never match without its host; a marker another module also
|
||||
decides on — the Facebook Chrome retry's FACEBOOK_PARSE_WALL — is imported, never retyped);
|
||||
3. pin the real text in `tests/test_download_errors.py`;
|
||||
4. add `downloads.errors.fail.catalog.<detail>.{short,body}` in EN and HU (a contract test checks it;
|
||||
the group is not called `detail`, which `actionErrors.contract` reads as a raw `HttpError.detail`).
|
||||
"""
|
||||
|
||||
# Two worker-authored strings that this module owns because both are coupled to `_RULES`.
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import NamedTuple
|
||||
|
||||
from app.downloads.adapters import (
|
||||
FACEBOOK_PARSE_WALL,
|
||||
POST_NOT_SHOWN_MESSAGE,
|
||||
POST_WITHOUT_VIDEO_MESSAGE,
|
||||
facebook_group_post_id,
|
||||
)
|
||||
from app.downloads.credentials import host_key_for_url
|
||||
|
||||
# Worker-authored strings that this module owns because each is coupled to `_RULES`.
|
||||
#
|
||||
# QUOTA_FULL_MESSAGE is raised by the worker and must classify back to `quota_exceeded`; its rule
|
||||
# below matches a fragment of it, and test_download_errors pins the pair, because a message and its
|
||||
# rule drifting apart is exactly how a category silently becomes unreachable.
|
||||
#
|
||||
# NETWORK_REDACTED_MESSAGE is the opposite direction: it REPLACES a message after classification, so
|
||||
# it deliberately has no rule. Nothing re-classifies a stored `job.error` today; if anything ever
|
||||
# does, it must carry the original code rather than re-deriving one from this text.
|
||||
# NETWORK_REDACTED_MESSAGE REPLACES a message after classification (`user_error_text`). It is written
|
||||
# to the shared asset too, and a second job that inherits that failed asset re-diagnoses the stored
|
||||
# text (`worker._process_job`), so it has a rule of its own that gives the category back.
|
||||
#
|
||||
# Other worker-raised texts are still inline literals at their raise sites. The ffmpeg ones and
|
||||
# "produced no output file" have markers here; the urlguard refusal deliberately has none and
|
||||
# classifies as `unknown` — it is a refusal, not a diagnosis, and no category would say more.
|
||||
#
|
||||
# A source adapter's verdicts on a link (`SourceAdapter.resolve_url`, E11.5 S4) are adapters.py's
|
||||
# constants, imported above; each has a rule below that gives its category back, and a catalog entry
|
||||
# that gives the site's sentence.
|
||||
QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
|
||||
NETWORK_REDACTED_MESSAGE = "The server could not reach that address."
|
||||
|
||||
@@ -30,6 +61,12 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
# Our own refusal: the worker found the user already at their storage cap (SEC-13). Placed
|
||||
# first — it is our wording, so it can never be a false match on an upstream message.
|
||||
("download storage is full", "quota_exceeded"),
|
||||
# Our own redaction of a transport failure (see NETWORK_REDACTED_MESSAGE) — our wording too.
|
||||
(NETWORK_REDACTED_MESSAGE.lower(), "network"),
|
||||
# Our own verdicts on a post link (adapters.py) — our wording too: its page holds no single video,
|
||||
# or the site answered the signed-in read with something that is not the post at all.
|
||||
(POST_WITHOUT_VIDEO_MESSAGE.lower(), "format_unavailable"),
|
||||
(POST_NOT_SHOWN_MESSAGE.lower(), "login_required"),
|
||||
# Login / age / members / bot-check — the user must be signed in (we deliberately are not).
|
||||
("sign in to confirm your age", "login_required"),
|
||||
("confirm your age", "login_required"),
|
||||
@@ -101,6 +138,120 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
)
|
||||
|
||||
|
||||
class Diagnosis(NamedTuple):
|
||||
"""What a failure was: the generic `code` (category) and, when a catalog entry matched, its
|
||||
host-specific `detail`."""
|
||||
|
||||
code: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Entry:
|
||||
"""One measured failure of one site. It matches when the download URL belongs to `host`
|
||||
(`credentials.SUPPORTED_HOSTS`), every one of `markers` is in the message, the cookie state is
|
||||
`cookie` (None: either), and — when `link` is given — it recognises the URL (a truthy result). A
|
||||
link predicate is the adapter's own, so the catalog and the download agree on what a link is."""
|
||||
|
||||
host: str
|
||||
markers: tuple[str, ...]
|
||||
code: str
|
||||
detail: str
|
||||
cookie: bool | None = None
|
||||
link: Callable[[str], object] | None = None
|
||||
|
||||
|
||||
_CATALOG: tuple[_Entry, ...] = (
|
||||
# Facebook answers a signed-out client with a page yt-dlp cannot parse — an 18+ reel, a closed
|
||||
# group's post (measured 2026-09-13: prod jobs 145, 156, 256 download once the owner's cookie is
|
||||
# attached). "Cannot parse data" is yt-dlp's generic wording for a broken extractor on EVERY site,
|
||||
# which is why the entry is scoped to Facebook and never becomes a general rule.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
||||
code="login_required",
|
||||
detail="facebook.login_wall",
|
||||
cookie=False,
|
||||
),
|
||||
# The same text WITH a cookie: an expired / signed-out cookie, or a page that also needs
|
||||
# impersonation (prod job 257). Telling this user to add cookies would be wrong — they have.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
||||
code="login_required",
|
||||
detail="facebook.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# A group post read with a cookie that Facebook answered with its login page (measured with a
|
||||
# refused cookie: no post id anywhere on the page) — the same refused cookie, found by the adapter.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=(POST_NOT_SHOWN_MESSAGE.lower(),),
|
||||
code="login_required",
|
||||
detail="facebook.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# A group post with no single video of its own. With a cookie the adapter reads the post's page
|
||||
# and says so itself (POST_WITHOUT_VIDEO_MESSAGE); without one yt-dlp is given the post link and
|
||||
# answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared
|
||||
# clips). With a cookie yt-dlp only ever gets the post's VIDEO link, so the same text there is about
|
||||
# that video, not the post — the category says it. Only for group post links (the adapter's own
|
||||
# test of one): a reel with no formats is a different failure.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),),
|
||||
code="format_unavailable",
|
||||
detail="facebook.group_post_video",
|
||||
link=facebook_group_post_id,
|
||||
),
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", "no video formats found"),
|
||||
code="format_unavailable",
|
||||
detail="facebook.group_post_video",
|
||||
cookie=False,
|
||||
link=facebook_group_post_id,
|
||||
),
|
||||
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
||||
# job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its
|
||||
# neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails
|
||||
# identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp).
|
||||
# The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so
|
||||
# "you're not a bot" with a straight quote never matches the real text.
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("[youtube]", "not a bot"),
|
||||
code="login_required",
|
||||
detail="youtube.bot_wall",
|
||||
cookie=False,
|
||||
),
|
||||
# The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first
|
||||
# one is being added now) — but "add cookies" is provably the wrong advice for someone who has,
|
||||
# and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no
|
||||
# longer valid. They have likely been rotated in the browser as a security measure."
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("[youtube]", "not a bot"),
|
||||
code="login_required",
|
||||
detail="youtube.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# The same gate one step later: the player response comes back, and then the CDN refuses the
|
||||
# stream itself ("unable to download video data: HTTP Error 403"). Measured 2026-09-16 on the
|
||||
# very same video, ~90 minutes after it was answering with the bot message — signed out it
|
||||
# 403'd on every attempt; with the owner's cookie attached it downloaded in full (519 MB). So
|
||||
# the category this text carries generically, `network`, is a lie here: nothing was wrong with
|
||||
# the network. Scoped to a cookie-less YouTube download, where that is what we measured.
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("unable to download video data", "403"),
|
||||
code="login_required",
|
||||
detail="youtube.media_forbidden",
|
||||
cookie=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def classify(msg: str | None) -> str:
|
||||
"""Return a category code for a raw failure message (``"unknown"`` if nothing matches)."""
|
||||
if not msg:
|
||||
@@ -112,6 +263,27 @@ def classify(msg: str | None) -> str:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def diagnose(
|
||||
msg: str | None, *, url: str | None = None, cookie_attached: bool = False
|
||||
) -> Diagnosis:
|
||||
"""The catalog entry for this failure of this link, else just its category (see the module doc).
|
||||
|
||||
`url` is the address yt-dlp was given; `cookie_attached` whether the acting user's cookie went
|
||||
with it. Without a URL only the category can be known."""
|
||||
host = host_key_for_url(url) if msg and url else None
|
||||
if host is not None:
|
||||
low = msg.lower() # type: ignore[union-attr] — `host` is only set when `msg` is truthy
|
||||
for e in _CATALOG:
|
||||
if (
|
||||
e.host == host
|
||||
and all(m in low for m in e.markers)
|
||||
and (e.cookie is None or e.cookie == cookie_attached)
|
||||
and (e.link is None or e.link(url)) # type: ignore[arg-type] — as `msg` above
|
||||
):
|
||||
return Diagnosis(e.code, e.detail)
|
||||
return Diagnosis(classify(msg))
|
||||
|
||||
|
||||
def user_error_text(source_kind: str, msg: str, code: str) -> str:
|
||||
"""The failure text to STORE on the job, given the raw message and its category.
|
||||
|
||||
|
||||
@@ -57,6 +57,10 @@ _DEFAULTS = {
|
||||
# opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped
|
||||
# the v3 net — must re-derive through the current browser-safe guarantee.
|
||||
_SIG_VERSION = 4
|
||||
# Audio-only downloads version on their own, so a change to the audio bytes doesn't re-download every
|
||||
# cached video. v5: an extracted .mp3/.m4a is stored as itself instead of remuxed into an .mp4 that
|
||||
# iPad's WebKit refuses to play, so a cached remuxed audio asset must re-derive.
|
||||
_AUDIO_SIG_VERSION = 5
|
||||
|
||||
_SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"]
|
||||
|
||||
@@ -96,20 +100,34 @@ def normalize(spec: dict) -> dict:
|
||||
return s
|
||||
|
||||
|
||||
def format_sig(spec: dict, salt: str | None = None) -> str:
|
||||
def _sig_version(s: dict) -> int:
|
||||
return _AUDIO_SIG_VERSION if s["mode"] == "a" else _SIG_VERSION
|
||||
|
||||
|
||||
def format_sig(spec: dict, salt: str | None = None, *, version: int | None = None) -> str:
|
||||
"""Stable short hash of the output-affecting spec — the MediaAsset cache key component.
|
||||
|
||||
`salt` forks a distinct, non-dedup identity for the same spec (used by force-redownload so it
|
||||
always re-fetches into its own asset instead of sharing a cached file with other jobs)."""
|
||||
always re-fetches into its own asset instead of sharing a cached file with other jobs).
|
||||
`version` hashes under a signature version other than the current one (see `format_sigs`)."""
|
||||
s = normalize(spec)
|
||||
data = {k: s[k] for k in _SIG_KEYS}
|
||||
data["_v"] = _SIG_VERSION
|
||||
data["_v"] = _sig_version(s) if version is None else version
|
||||
if salt is not None:
|
||||
data["_salt"] = salt
|
||||
payload = json.dumps(data, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(payload.encode()).hexdigest()[:24]
|
||||
|
||||
|
||||
def format_sigs(spec: dict, salt: str | None = None) -> tuple[str, str]:
|
||||
"""`format_sig` under the current signature version and under the one before it. An asset keeps
|
||||
the signature it was created with, so a job queued before a deploy that bumped the version and
|
||||
run after it downloads into an asset carrying the previous one — which must still be recognised
|
||||
as that user's private asset (`credentials.is_authed_asset`), or it downloads without its cookie."""
|
||||
previous = _sig_version(normalize(spec)) - 1
|
||||
return format_sig(spec, salt), format_sig(spec, salt, version=previous)
|
||||
|
||||
|
||||
def build_ydl_opts(
|
||||
spec: dict,
|
||||
outtmpl: str,
|
||||
|
||||
@@ -8,7 +8,6 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
@@ -114,9 +113,10 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
# `or info.get("id")` avoids a blank card while downloading when the cleaned title is empty
|
||||
# (e.g. a title that was only a social engagement prefix); the completion path later sets
|
||||
# the final title with its own source_ref fallback.
|
||||
asset.title = normalize_title(info.get("title")) or info.get("id")
|
||||
adapter = adapters.resolve_adapter(None, None, info)
|
||||
asset.title = normalize_title(adapter.title(info)) or info.get("id")
|
||||
asset.uploader = info.get("uploader") or info.get("channel")
|
||||
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
|
||||
asset.uploader_url = adapter.uploader_url(info)
|
||||
asset.thumbnail_url = info.get("thumbnail")
|
||||
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
||||
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
||||
@@ -323,14 +323,33 @@ def _extract_with_fallback(
|
||||
):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt.
|
||||
error); a Facebook download WITH a cookie is retried as Chrome behind "Cannot parse data"; other
|
||||
sources make a single attempt. Returns the info dict of the successful attempt.
|
||||
|
||||
Before the first attempt the adapter may resolve `url` to the link that really holds the video (a
|
||||
Facebook group post → its video's watch link); when the page shows no single video there, or is not
|
||||
the post at all (a refused cookie's login page), the adapter raises its own verdict
|
||||
(adapters.POST_*_MESSAGE) instead of letting yt-dlp guess, and a failed page read fails the
|
||||
download as itself.
|
||||
|
||||
`cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated.
|
||||
|
||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt."""
|
||||
client_sets = adapter.client_sets()
|
||||
client_sets = adapter.client_sets(authed=cookiefile is not None)
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
# Resolving may read a page first — network time the stall watchdog must not see as a stall.
|
||||
_touch_job(job_id)
|
||||
target = adapter.resolve_url(
|
||||
url,
|
||||
formats.build_ydl_opts(
|
||||
spec,
|
||||
outtmpl,
|
||||
_make_progress_hook(job_id, asset_id),
|
||||
cookiefile=cookiefile,
|
||||
max_bytes=max_bytes,
|
||||
),
|
||||
)
|
||||
last_exc: Exception | None = None
|
||||
for i, clients in enumerate(client_sets):
|
||||
opts = formats.build_ydl_opts(
|
||||
@@ -348,7 +367,7 @@ def _extract_with_fallback(
|
||||
# killed — acceptable, as socket_timeout + extractor_retries bound one attempt well under it.)
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||
return ydl.extract_info(url, download=True)
|
||||
return ydl.extract_info(target, download=True)
|
||||
except yt_dlp.utils.DownloadError as exc:
|
||||
last_exc = exc
|
||||
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||
@@ -385,8 +404,12 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
or just remux when the codecs are fine but the container isn't. **Scope:** only the mp4 compat
|
||||
profile is normalized; an explicit `vcodec` or a non-mp4 custom profile is the user opting out
|
||||
of the guarantee and is left as-is (so a `webm` profile can still ship VP9/Opus). Returns the
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk."""
|
||||
if spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk.
|
||||
|
||||
An audio download (mode "a") is never touched: yt-dlp already extracted it to .mp3 / .m4a, which
|
||||
every browser plays as audio/mpeg / audio/mp4. Remuxing it into .mp4 served an mp3 as video/mp4,
|
||||
which iPad's WebKit refuses to play (prod job 268)."""
|
||||
if spec.get("mode") == "a" or spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
return produced
|
||||
vcodec, acodec = editmod.probe_codecs(produced)
|
||||
is_mp4 = produced.suffix.lower() == ".mp4"
|
||||
@@ -425,6 +448,7 @@ def _download(
|
||||
) -> None:
|
||||
staging = _staging_dir(asset_id)
|
||||
cookiefile: Path | None = None
|
||||
url: str | None = None # read by the failure branch, which can run before it is resolved
|
||||
try:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
@@ -445,12 +469,10 @@ def _download(
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
remaining_bytes: int | None = None
|
||||
with SessionLocal() as cdb:
|
||||
asset_sig = cdb.execute(
|
||||
select(MediaAsset.format_sig).where(MediaAsset.id == asset_id)
|
||||
).scalar_one_or_none()
|
||||
if asset_sig is not None:
|
||||
target = cdb.get(MediaAsset, asset_id)
|
||||
if target is not None:
|
||||
cookiefile = credentials.resolve_cookiefile(
|
||||
cdb, user_id, url, asset_sig, spec, staging
|
||||
cdb, user_id, url, target, spec, staging
|
||||
)
|
||||
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
|
||||
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a
|
||||
@@ -484,13 +506,19 @@ def _download(
|
||||
produced = _produced_file(info, staging)
|
||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||
ext = produced.suffix.lstrip(".").lower()
|
||||
# Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's
|
||||
# `info` dict, whose fields are absent for some sources (Facebook among them): deriving
|
||||
# media_kind from those nulls misclassified a real H.264 video as a download-only 'file', and a
|
||||
# missing duration hid the trim action. Probed before the sidecars so the .nfo runtime has it
|
||||
# too. Same probe the upload ingest uses.
|
||||
vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
|
||||
meta = storage.MediaMeta(
|
||||
video_id=info.get("id") or source_ref,
|
||||
title=normalize_title(info.get("title")) or source_ref,
|
||||
title=normalize_title(adapter.title(info)) or source_ref,
|
||||
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
||||
upload_date=_parse_upload_date(info.get("upload_date")),
|
||||
uniqueid_type=adapter.nfo_uniqueid_type(),
|
||||
duration_s=int(info["duration"]) if info.get("duration") else None,
|
||||
duration_s=duration_s,
|
||||
description=info.get("description"),
|
||||
thumbnail_url=info.get("thumbnail"),
|
||||
)
|
||||
@@ -516,12 +544,11 @@ def _download(
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = size
|
||||
asset.duration_s = meta.duration_s
|
||||
# Codecs/dims come from a PROBE of the finished file — the ground truth — NOT yt-dlp's
|
||||
# `info` dict, whose codec fields are absent for some sources (Facebook among them):
|
||||
# deriving media_kind from those nulls misclassified a real H.264 video as a
|
||||
# download-only 'file' with no in-app play/edit. Same probe the upload ingest uses.
|
||||
asset.vcodec, asset.acodec, asset.width, asset.height = probe.resolve_media_facts(
|
||||
final, info
|
||||
asset.vcodec, asset.acodec, asset.width, asset.height = (
|
||||
vcodec,
|
||||
acodec,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
asset.container = ext
|
||||
asset.media_kind = mediakind.derive(asset.vcodec, asset.acodec)
|
||||
@@ -578,6 +605,7 @@ def _download(
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
error_detail=None,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
@@ -588,9 +616,11 @@ def _download(
|
||||
log.info("job %s refused for its owner: %s", job_id, msg)
|
||||
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
|
||||
msg = str(exc)[:500]
|
||||
code = dlerrors.classify(msg)
|
||||
# The catalog needs the link and whether the user's cookie went with it: the same yt-dlp
|
||||
# text is Facebook's login wall without one and a refused cookie with one.
|
||||
diagnosis = dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
|
||||
# SEC-12: redact a transport-level readout for user-supplied URLs (see user_error_text).
|
||||
msg = dlerrors.user_error_text(source_kind, msg, code)
|
||||
msg = dlerrors.user_error_text(source_kind, msg, diagnosis.code)
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
@@ -603,7 +633,8 @@ def _download(
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=code,
|
||||
error_code=diagnosis.code,
|
||||
error_detail=diagnosis.detail,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
|
||||
@@ -81,10 +81,12 @@ def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | N
|
||||
|
||||
def resolve_media_facts(
|
||||
path: Path, info: dict
|
||||
) -> tuple[str | None, str | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height) for the stored download asset: PROBE the finished file on disk
|
||||
(the ground truth). yt-dlp's `info` omits codec/dimension fields for some sources (Facebook among
|
||||
them), so deriving media_kind from it there labelled a real H.264 video a download-only 'file'.
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) for the stored download asset: PROBE the finished
|
||||
file on disk (the ground truth). yt-dlp's `info` omits codec/dimension fields for some sources
|
||||
(Facebook among them), so deriving media_kind from it there labelled a real H.264 video a
|
||||
download-only 'file'. It omits the duration too (a Facebook reel, an Instagram reel), and a NULL
|
||||
duration hides the Library row's trim action although the file plays for minutes.
|
||||
|
||||
Precedence is by whether the probe RAN, not per-field: if ffprobe read the file, its codec verdict
|
||||
is authoritative — INCLUDING a deliberate None (cover-art-only audio, a still image). A per-field
|
||||
@@ -92,16 +94,23 @@ def resolve_media_facts(
|
||||
have (yt-dlp reports a progressive `vcodec` for an audio-mode download whose video it then strips),
|
||||
mislabelling an audio download as a broken in-app 'video'. Only when ffprobe could NOT read the
|
||||
file at all do we fall back to `info` (the pre-probe behaviour, a safety net); the cosmetic
|
||||
dimensions still gap-fill from `info` in either case."""
|
||||
dimensions and the duration still gap-fill from `info` in either case."""
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
info_v = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||
info_a = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
info_w = int(info["width"]) if info.get("width") else None
|
||||
info_h = int(info["height"]) if info.get("height") else None
|
||||
info_d = int(info["duration"]) if info.get("duration") else None
|
||||
if data is None:
|
||||
# ffprobe could not read the file — fall back to the extractor's info entirely.
|
||||
return info_v, info_a, info_w, info_h
|
||||
pv, pa, pw, ph, _dur = _facts_from_probe(data)
|
||||
return info_v, info_a, info_w, info_h, info_d
|
||||
pv, pa, pw, ph, pd = _facts_from_probe(data)
|
||||
# Probe ran → its codecs win outright (a None means "no such track", not "unknown"); info only
|
||||
# gap-fills the cosmetic width/height the probe couldn't read.
|
||||
return pv, pa, pw if pw is not None else info_w, ph if ph is not None else info_h
|
||||
# gap-fills the width/height/duration the probe couldn't read.
|
||||
return (
|
||||
pv,
|
||||
pa,
|
||||
pw if pw is not None else info_w,
|
||||
ph if ph is not None else info_h,
|
||||
pd if pd is not None else info_d,
|
||||
)
|
||||
|
||||
@@ -17,6 +17,8 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from app import logs
|
||||
from app import logsink
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
@@ -52,12 +54,16 @@ def _deprioritize() -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logs.configure_logging("runner")
|
||||
_deprioritize()
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: python -m app.downloads.runner <job_id>", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
job_id = int(sys.argv[1])
|
||||
# This whole process exists to run ONE job — tag every line it prints with it, so the child's
|
||||
# output joins up with the worker's and the API's view of the same download.
|
||||
logs.set_job(job_id)
|
||||
logsink.start() # flushed at exit (atexit) — this process has no shutdown hook of its own
|
||||
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
|
||||
@@ -7,6 +7,7 @@ handles everything else. State transitions used by the routes (pause/resume/canc
|
||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
@@ -80,11 +81,13 @@ def get_or_create_asset(
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
try:
|
||||
db.flush()
|
||||
# A SAVEPOINT: losing the race undoes only this insert, never the caller's own pending writes
|
||||
# (Retry clears the job's error before it gets here; a whole-session rollback threw that away).
|
||||
with db.begin_nested():
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
asset = db.execute(stmt).scalar_one()
|
||||
return asset
|
||||
|
||||
@@ -341,13 +344,20 @@ def pause(db: Session, job: DownloadJob) -> None:
|
||||
def resume(db: Session, job: DownloadJob) -> None:
|
||||
"""Requeue a paused/errored job for a fresh attempt, clearing its error. If the SHARED asset itself
|
||||
failed, reset it to pending so the worker actually re-downloads — otherwise the requeued job would
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset)."""
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset).
|
||||
|
||||
A cookie the user added AFTER the job was queued moves the job onto their private asset first
|
||||
(`_follow_the_cookie`): the worker only ever attaches a cookie to that identity, so without the move
|
||||
the Retry a login-wall message asks for would fail on the same wall."""
|
||||
if job.status in (JobStatus.PAUSED, JobStatus.ERROR):
|
||||
job.progress = 0
|
||||
job.error = None
|
||||
job.error_code = None
|
||||
job.error_detail = None
|
||||
if job.asset_id:
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
if asset is not None:
|
||||
asset = _follow_the_cookie(db, job, asset)
|
||||
if asset and asset.status == "error": # MediaAsset.status (no enum), not a JobStatus
|
||||
asset.status = "pending"
|
||||
asset.error = None
|
||||
@@ -355,6 +365,32 @@ def resume(db: Session, job: DownloadJob) -> None:
|
||||
db.commit()
|
||||
|
||||
|
||||
def _follow_the_cookie(db: Session, job: DownloadJob, asset: MediaAsset) -> MediaAsset:
|
||||
"""Re-point a requeued job at the asset enqueue would pick NOW, when the user has a cookie for the
|
||||
link's host but the job still holds an asset the worker won't attach it to (queued before the
|
||||
cookie existed). Returns the asset the job holds afterwards.
|
||||
|
||||
Only a `pending` / `error` asset moves — neither has a file nor a download in flight. A `ready` one
|
||||
already has the file, and a `downloading` one may be this job's own child still being stopped by
|
||||
the pause; the next Retry after it settles moves the job then. Removing a cookie never moves a job
|
||||
back: the worker attaches nothing without a credential, so the private asset is as good as any."""
|
||||
if asset.status not in ("pending", "error"):
|
||||
return asset
|
||||
url = source_url(job.source_kind, job.source_ref)
|
||||
salt = credentials.salt_for_enqueue(db, job.user_id, url)
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
if salt is None or credentials.is_authed_asset(asset, spec, job.user_id, url):
|
||||
return asset
|
||||
target = get_or_create_asset(
|
||||
db, job.source_kind, job.source_ref, formats.format_sig(spec, salt=salt)
|
||||
)
|
||||
populate_from_catalog(db, target)
|
||||
release_asset(db, job) # drop the old hold BEFORE re-pointing — it keys off job.asset_id
|
||||
incr_ref_count(db, target.id)
|
||||
job.asset_id = target.id
|
||||
return target
|
||||
|
||||
|
||||
def cancel(db: Session, job: DownloadJob) -> None:
|
||||
"""Cancel a job unless it's already done/canceled, releasing its asset hold WHILE it still counts
|
||||
as holding — `canceled` ∉ HOLDING, so a later delete won't double-release."""
|
||||
@@ -373,13 +409,31 @@ def delete(db: Session, job: DownloadJob) -> None:
|
||||
|
||||
def redownload(db: Session, job: DownloadJob) -> None:
|
||||
"""Rebuild a finished/failed job's file into a FRESH, private asset (the cache sig is salted with
|
||||
the job id + current asset id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
the new asset's own id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
so any public share link (DownloadLink -> job_id) keeps resolving to the newly-downloaded file.
|
||||
The old hold is dropped first (freeing the old file iff this job was its last holder). The caller
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409)."""
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409).
|
||||
|
||||
When the user has a cookie for the link's host, the salt carries it (`credentials.rebuild_salt`),
|
||||
so the worker recognises the rebuild as their private identity and attaches the cookie — a cookie
|
||||
download, or one the user added after a failure, rebuilds with the cookie."""
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
sig = formats.format_sig(spec, salt=f"{job.id}:{job.asset_id}")
|
||||
asset = get_or_create_asset(db, job.source_kind, job.source_ref, sig)
|
||||
cookie = credentials.salt_for_enqueue(
|
||||
db, job.user_id, source_url(job.source_kind, job.source_ref)
|
||||
)
|
||||
# The salt names the row's own id, which exists only once the row does: insert it under a unique
|
||||
# placeholder, then write the real signature in the same transaction (nobody sees the placeholder).
|
||||
asset = MediaAsset(
|
||||
source_kind=job.source_kind,
|
||||
source_ref=job.source_ref,
|
||||
format_sig=f"rebuild:{uuid.uuid4().hex}",
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
asset.format_sig = formats.format_sig(spec, salt=credentials.rebuild_salt(asset.id, cookie))
|
||||
db.flush()
|
||||
populate_from_catalog(db, asset)
|
||||
# Drop the old hold BEFORE re-pointing the job — release_asset keys off the job's current asset.
|
||||
release_asset(db, job)
|
||||
@@ -389,6 +443,8 @@ def redownload(db: Session, job: DownloadJob) -> None:
|
||||
job.progress = 0
|
||||
job.phase = None
|
||||
job.error = None
|
||||
job.error_code = None # a rebuild that fails again gets its own diagnosis, never the old one
|
||||
job.error_detail = None
|
||||
job.speed_bps = None
|
||||
job.eta_s = None
|
||||
# Re-derive the display name from the freshly cleaned title on completion (else an older, un-cleaned
|
||||
|
||||
@@ -117,6 +117,19 @@ def tags_by_category(db: Session, tag_ids: list[int]) -> dict[str, list[int]]:
|
||||
return out
|
||||
|
||||
|
||||
def not_blocked_cond(user: User):
|
||||
"""Per-user channel blocklist: never show this user videos from a channel they blocked
|
||||
(applies to feed, Library, explore, the channel page and the Home resume rail alike).
|
||||
|
||||
A shared expression rather than a clause each surface writes for itself. Blocking does not touch
|
||||
`video_states`, so a half-watched video from a newly blocked channel survives with its resume
|
||||
position — and the one query that copied only PART of this predicate set is exactly how that
|
||||
video came back as the landing page's suggestion after the user had said "never again"."""
|
||||
return Video.channel_id.notin_(
|
||||
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
|
||||
)
|
||||
|
||||
|
||||
def filtered_query(
|
||||
db: Session,
|
||||
user: User,
|
||||
@@ -207,13 +220,7 @@ def filtered_query(
|
||||
)
|
||||
)
|
||||
|
||||
# Per-user channel blocklist: never show this user videos from a channel they blocked
|
||||
# (applies to feed, Library, explore, and the channel page alike).
|
||||
query = query.where(
|
||||
Video.channel_id.notin_(
|
||||
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
|
||||
)
|
||||
)
|
||||
query = query.where(not_blocked_cond(user))
|
||||
|
||||
# Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
|
||||
# (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
|
||||
|
||||
@@ -0,0 +1,685 @@
|
||||
"""Structured logging for every Siftlode process — the ONE place logging is configured.
|
||||
|
||||
Before this module a single deployment printed FOUR different log formats: uvicorn's
|
||||
`log_config.json` for the API, a hand-rolled StreamHandler in `main.py` for everything that did not
|
||||
start under uvicorn (tests, scripts), `logging.basicConfig()` in the worker and in the download
|
||||
child (no timestamp at all), and alembic's `fileConfig`. None of them carried a correlation id, and
|
||||
uvicorn's access log printed the query string of every request (`?account=1`, `?source=feed`) — so
|
||||
the logs were both unjoinable and leaky. `configure_logging()` replaces all four.
|
||||
|
||||
Event shape — the OpenTelemetry log data model, without the SDK; one JSON object per line:
|
||||
ts · level · sev · source · logger · event · msg · attrs
|
||||
plus whatever correlation the context carries (`request_id`, `user_id`, `debug_session_id`,
|
||||
`job_id`) and the app version. That shape is what E22.6's admin console filters on and what a Gitea
|
||||
issue attachment will carry, so it is deliberately flat and stable.
|
||||
|
||||
Call sites stay ordinary stdlib logging:
|
||||
|
||||
log = logging.getLogger("siftlode.downloads")
|
||||
log.info("job finished", extra=logs.event("download.job.done", job_id=7, bytes=n))
|
||||
|
||||
`event()` is optional — a bare `log.info("…")` still comes out well-formed — but the dotted code is
|
||||
what makes a line queryable later, so add one wherever the line matters.
|
||||
|
||||
REDACTION IS APPLIED AT THE HANDLER, not at the call site: every message, every attr value and
|
||||
every traceback passes `redact()` on the way out. That is the whole point — a secret cannot be
|
||||
leaked by forgetting to redact it at one of ~125 call sites, and a new call site is safe by
|
||||
default. Never logged, at any level: passwords, tokens (session, reset/verify, setup, share,
|
||||
OAuth), cookie blobs, secret config values, and email addresses.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import logging.config
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from app.config import settings
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
|
||||
# One level below DEBUG for the "Teljes / TRACE" diagnostics level (search strings, redacted request
|
||||
# bodies). Registered on import so `logging.getLogger(...).log(logs.TRACE, …)` and a `LOG_LEVEL=TRACE`
|
||||
# both work in every process, including one that never calls configure_logging (a test).
|
||||
TRACE = 5
|
||||
logging.addLevelName(TRACE, "TRACE")
|
||||
|
||||
# OpenTelemetry severity numbers for our level names. Kept beside the level because the number is
|
||||
# what a log backend sorts and filters on; the name is for humans reading a terminal.
|
||||
SEVERITY: dict[str, int] = {
|
||||
"TRACE": 1,
|
||||
"DEBUG": 5,
|
||||
"INFO": 9,
|
||||
"WARNING": 13,
|
||||
"ERROR": 17,
|
||||
"CRITICAL": 21,
|
||||
}
|
||||
|
||||
# Which process this is. Set by configure_logging; every line carries it so one Loki/DB query can
|
||||
# separate the API's view of a job from the worker's.
|
||||
_source = "api"
|
||||
|
||||
|
||||
# --- correlation context -------------------------------------------------------------------
|
||||
# contextvars, like quota's actor attribution (app/quota.py): set at an entry point, read by the
|
||||
# formatter, never threaded through call signatures. A child task copies the context at creation,
|
||||
# so anything set before `await call_next(...)` is visible to the whole request.
|
||||
_request_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"log_request_id", default=None
|
||||
)
|
||||
_user_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_user_id", default=None)
|
||||
_debug_session: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"log_debug_session", default=None
|
||||
)
|
||||
_job_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_job_id", default=None)
|
||||
|
||||
_CONTEXT_VARS = {
|
||||
"request_id": _request_id,
|
||||
"user_id": _user_id,
|
||||
"debug_session_id": _debug_session,
|
||||
"job_id": _job_id,
|
||||
}
|
||||
|
||||
|
||||
class _RequestScope:
|
||||
"""What the auth dependency learns about a request, for every line logged after it.
|
||||
|
||||
A MUTABLE object behind a contextvar, not plain contextvars, because of how a request actually
|
||||
runs: FastAPI resolves a sync dependency (`auth.current_user`) in a threadpool on a COPY of the
|
||||
context, so a `ContextVar.set()` there is gone by the time the handler runs — measured: that is
|
||||
how the account never reached a handler's own lines. `log_requests` installs one scope per
|
||||
request; every copied context shares the same object, so what the dependency writes is seen by
|
||||
the handler and by the request line itself."""
|
||||
|
||||
__slots__ = ("user_id", "level")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.user_id: int | None = None
|
||||
# The user's consented diagnostics level as a logging number; None at the base level.
|
||||
self.level: int | None = None
|
||||
|
||||
|
||||
_scope: contextvars.ContextVar[_RequestScope | None] = contextvars.ContextVar(
|
||||
"log_request_scope", default=None
|
||||
)
|
||||
|
||||
|
||||
def set_user(user_id: int | None, level: str = "INFO", until: datetime | None = None) -> None:
|
||||
"""Attribute the rest of this request's log lines to `user_id`, and log them at the user's
|
||||
consented diagnostics `level` until `until` (E22 S5; "INFO" = no elevation). Called from the
|
||||
auth dependency — the one place the session is actually resolved into an account — rather than
|
||||
from the request middleware, which runs OUTSIDE the session middleware."""
|
||||
_user_id.set(user_id)
|
||||
scope = _scope.get()
|
||||
if scope is None: # outside log_requests (a script, a unit test): this context only
|
||||
scope = _RequestScope()
|
||||
_scope.set(scope)
|
||||
elevated = level != "INFO" and until is not None and until > datetime.now(timezone.utc)
|
||||
scope.user_id = user_id
|
||||
scope.level = resolve_level(level) if elevated else None
|
||||
if user_id is not None:
|
||||
_track_elevation(user_id, scope.level, until)
|
||||
|
||||
|
||||
def set_job(job_id: int | None) -> None:
|
||||
"""Attribute this process's log lines to a download job — for the download CHILD, whose whole
|
||||
life is one job (the worker uses `bind()` per job instead, since it juggles several)."""
|
||||
_job_id.set(job_id)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def bind(**fields: Any):
|
||||
"""Bind correlation fields for the duration of a block (a worker job, a scheduler run):
|
||||
|
||||
with logs.bind(job_id=job.id):
|
||||
...
|
||||
|
||||
Unknown field names are ignored rather than raising — a caller binding a field this module does
|
||||
not know about is a typo, not a reason to take down a download."""
|
||||
tokens = [(_CONTEXT_VARS[k].set(v)) for k, v in fields.items() if k in _CONTEXT_VARS]
|
||||
keys = [k for k in fields if k in _CONTEXT_VARS]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for key, token in zip(keys, tokens):
|
||||
_CONTEXT_VARS[key].reset(token)
|
||||
|
||||
|
||||
def event(code: str, **attrs: Any) -> dict[str, Any]:
|
||||
"""`extra=` payload for a log call: a stable dotted event code plus structured attributes.
|
||||
|
||||
Returns a plain dict so it composes with stdlib logging and costs nothing when the level is off
|
||||
(the dict is built either way; keep the attrs cheap — no DB reads inside the call)."""
|
||||
return {"event": code, "attrs": attrs}
|
||||
|
||||
|
||||
# --- redaction -----------------------------------------------------------------------------
|
||||
# Substring matching, not exact keys: the same secret travels as `password`, `current_password`,
|
||||
# `new_password`, `smtp_password`, `plex_server_token`, `access_token`, … Matching on the PART is
|
||||
# what makes a key we have never seen safe by default.
|
||||
_SECRET_KEY_PARTS = (
|
||||
"password",
|
||||
"passphrase",
|
||||
"token",
|
||||
"secret",
|
||||
"cookie",
|
||||
"authorization",
|
||||
"auth_header",
|
||||
"ciphertext",
|
||||
"private_key",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"credential",
|
||||
"session_key",
|
||||
)
|
||||
|
||||
# Query/fragment parameters that carry a credential in a URL we log (OAuth `code`/`state`, our own
|
||||
# `?token=` setup and share links, `#verify=` / `#reset=` fragments).
|
||||
_SECRET_PARAM_NAMES = (
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"id_token",
|
||||
"token",
|
||||
"code",
|
||||
"state",
|
||||
"key",
|
||||
"secret",
|
||||
"password",
|
||||
"pwd",
|
||||
"sig",
|
||||
"signature",
|
||||
"verify",
|
||||
"reset",
|
||||
"setup",
|
||||
)
|
||||
_SECRET_PARAM_RE = re.compile(
|
||||
r"\b((?:" + "|".join(_SECRET_PARAM_NAMES) + r")=)([^&\s\"']+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Deliberately loose: over-matching redacts a non-address that looks like one (harmless), while
|
||||
# under-matching puts a real address in a log we attach to an issue (not harmless).
|
||||
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
REDACTED = "[redacted]"
|
||||
# `extra={PERSIST: False}` keeps a line out of the `log_events` table while stdout still gets it —
|
||||
# how `log_requests` caps anonymous traffic per client (app/logsink.py reads it).
|
||||
PERSIST = "persist"
|
||||
REDACTED_EMAIL = "[email]"
|
||||
|
||||
|
||||
def redact_text(text: str) -> str:
|
||||
"""Scrub a free-text string: credential-bearing URL params and email addresses."""
|
||||
scrubbed = _SECRET_PARAM_RE.sub(lambda m: f"{m.group(1)}{REDACTED}", text)
|
||||
return _EMAIL_RE.sub(REDACTED_EMAIL, scrubbed)
|
||||
|
||||
|
||||
def redact_query(query: str) -> str:
|
||||
"""Scrub a raw query string. DECODED first: the text rules match what was typed, and a browser
|
||||
sends it percent-encoded — `someone%40example.com` is an address `_EMAIL_RE` cannot see. Each
|
||||
parameter also goes through the key rules, so `?new_password=` goes as `redact` would drop it."""
|
||||
pairs = parse_qsl(query, keep_blank_values=True)
|
||||
return "&".join(f"{key}={_redact_param(key, value)}" for key, value in pairs)
|
||||
|
||||
|
||||
def _redact_param(key: str, value: str) -> str:
|
||||
"""One decoded `key=value` pair. The parameter-NAME rule is applied to the key here rather than
|
||||
left to `_SECRET_PARAM_RE` over the rebuilt pair: that pattern was written for the raw,
|
||||
percent-encoded query, where a value holds no literal space, `&` or quote, and its value class
|
||||
stops at the first one. On decoded text `?state=a%20b` would keep its tail and `?code=%20abc`
|
||||
would not match at all — the whole credential straight into the line."""
|
||||
if key.lower() in _SECRET_PARAM_NAMES:
|
||||
return REDACTED
|
||||
return str(redact(value, key=key))
|
||||
|
||||
|
||||
def redact(value: Any, *, key: str = "") -> Any:
|
||||
"""Recursively scrub a structure destined for the log.
|
||||
|
||||
A value whose KEY names a secret is dropped whole (we never log a prefix of a token — a prefix
|
||||
is still a leak and still useless). Everything else is scrubbed as text. Depth is bounded by
|
||||
the structures we actually log (attrs dicts), and non-JSON types fall through to `str`."""
|
||||
lowered = key.lower()
|
||||
if key and any(part in lowered for part in _SECRET_KEY_PARTS):
|
||||
return REDACTED
|
||||
if isinstance(value, dict):
|
||||
return {k: redact(v, key=str(k)) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [redact(v, key=key) for v in value]
|
||||
if isinstance(value, str):
|
||||
return redact_text(value)
|
||||
return value
|
||||
|
||||
|
||||
# --- formatters ----------------------------------------------------------------------------
|
||||
# Attributes every LogRecord carries; anything else a caller passed via `extra=` is a structured
|
||||
# attribute and is merged into `attrs` (so `extra={"job_id": 7}` works as well as `event(...)`).
|
||||
_RECORD_STANDARD = frozenset(
|
||||
vars(logging.LogRecord("", 0, "", 0, "", None, None)).keys()
|
||||
| {"message", "asctime", "taskName", "event", "attrs", PERSIST, "elevated"}
|
||||
)
|
||||
|
||||
|
||||
class ContextFilter(logging.Filter):
|
||||
"""Copies the correlation contextvars onto the record. A Filter (not the formatter) so the
|
||||
values are captured at emit time, before a QueueHandler can move the record to another
|
||||
thread — the DB sink in S3 depends on that."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
for name, var in _CONTEXT_VARS.items():
|
||||
value = var.get()
|
||||
if value is not None and not hasattr(record, name):
|
||||
setattr(record, name, value)
|
||||
scope = _scope.get()
|
||||
if scope is not None and scope.user_id is not None and not hasattr(record, "user_id"):
|
||||
record.user_id = scope.user_id
|
||||
return True
|
||||
|
||||
|
||||
class LevelGate(logging.Filter):
|
||||
"""The configured level, enforced at the HANDLER, with one exception: the lines of a request
|
||||
whose user raised their diagnostics level (E22 S5).
|
||||
|
||||
The root logger's level cannot do this alone: it is one number for the whole process, and the
|
||||
point is to log ONE user's requests in detail while everyone else's stay at the base level. So
|
||||
the root is lowered to the lowest level any active elevation needs (`_track_elevation`), and
|
||||
this gate drops what only an elevation let through unless the line belongs to that elevated
|
||||
request. It also marks such a line `elevated`, which is what makes the `log_events` sink keep
|
||||
every line of that request, not just its warnings."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
scope = _scope.get()
|
||||
if scope is not None and scope.level is not None:
|
||||
record.elevated = True
|
||||
return record.levelno >= min(scope.level, _base_level)
|
||||
return record.levelno >= _base_level
|
||||
|
||||
|
||||
def _record_attrs(record: logging.LogRecord) -> dict[str, Any]:
|
||||
extra = {k: v for k, v in vars(record).items() if k not in _RECORD_STANDARD}
|
||||
attrs = dict(getattr(record, "attrs", None) or {})
|
||||
attrs.update({k: v for k, v in extra.items() if k not in _CONTEXT_VARS})
|
||||
return attrs
|
||||
|
||||
|
||||
def _safe_message(record: logging.LogRecord) -> str:
|
||||
"""`record.getMessage()` raises if the args don't match the format string. A broken log call
|
||||
must not take down the caller, so the failure is reported IN the line instead."""
|
||||
try:
|
||||
return record.getMessage()
|
||||
except Exception as exc: # noqa: BLE001 — any formatting error, reported not raised
|
||||
return f"{record.msg!r} (log formatting failed: {exc})"
|
||||
|
||||
|
||||
def record_fields(record: logging.LogRecord) -> dict[str, Any]:
|
||||
"""The event as structured fields, REDACTED — the one definition both the stdout line and the
|
||||
`log_events` row are built from, so the two sinks can never disagree about what is safe to keep.
|
||||
Absent optional fields are left out; `ts` is an aware datetime."""
|
||||
fields: dict[str, Any] = {
|
||||
"ts": datetime.fromtimestamp(record.created, timezone.utc),
|
||||
"level": record.levelname,
|
||||
"sev": SEVERITY.get(record.levelname, 9),
|
||||
"source": _source,
|
||||
"logger": record.name,
|
||||
"msg": redact_text(_safe_message(record)),
|
||||
}
|
||||
code = getattr(record, "event", None)
|
||||
if code:
|
||||
fields["event"] = code
|
||||
for name in _CONTEXT_VARS:
|
||||
value = getattr(record, name, None)
|
||||
if value is not None:
|
||||
fields[name] = value
|
||||
attrs = _record_attrs(record)
|
||||
if attrs:
|
||||
fields["attrs"] = redact(attrs)
|
||||
if record.exc_info:
|
||||
fields["exc"] = redact_text(logging.Formatter().formatException(record.exc_info))
|
||||
elif record.exc_text:
|
||||
# A QueueHandler-style hand-off formats the traceback into exc_text and drops exc_info.
|
||||
fields["exc"] = redact_text(record.exc_text)
|
||||
fields["version"] = settings.app_version
|
||||
return fields
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line — the shape Loki and the `log_events` table both ingest."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload = record_fields(record)
|
||||
payload["ts"] = payload["ts"].isoformat(timespec="milliseconds")
|
||||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
class TextFormatter(logging.Formatter):
|
||||
"""Human-readable single line for a terminal (`LOG_FORMAT=text`): the same content, ordered for
|
||||
reading. Redaction applies here too — a dev log is still a log someone pastes into an issue."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
stamp = datetime.fromtimestamp(record.created, timezone.utc).strftime("%H:%M:%S.%f")[:-3]
|
||||
head = f"{stamp} {record.levelname:<5} [{record.name}]"
|
||||
rid = getattr(record, "request_id", None)
|
||||
if rid:
|
||||
head += f" {rid}"
|
||||
code = getattr(record, "event", None)
|
||||
line = f"{head} {code + ' — ' if code else ''}{redact_text(_safe_message(record))}"
|
||||
attrs = _record_attrs(record)
|
||||
if attrs:
|
||||
line += f" {json.dumps(redact(attrs), ensure_ascii=False, default=str)}"
|
||||
if record.exc_info:
|
||||
line += "\n" + redact_text(self.formatException(record.exc_info))
|
||||
return line
|
||||
|
||||
|
||||
# --- configuration -------------------------------------------------------------------------
|
||||
# Third-party loggers that are noisy at our own level. Pinned explicitly so the diagnostics knob can
|
||||
# lower the ROOT level to DEBUG/TRACE without drowning the output in SQL statements and HTTP chatter
|
||||
# — raising THESE is a deliberate, separate act (E22.6's console).
|
||||
_THIRD_PARTY_LEVELS = {
|
||||
"sqlalchemy": "WARNING",
|
||||
"sqlalchemy.engine": "WARNING",
|
||||
"apscheduler": "WARNING",
|
||||
"urllib3": "WARNING",
|
||||
"httpx": "WARNING",
|
||||
"httpcore": "WARNING",
|
||||
"asyncio": "WARNING",
|
||||
"multipart": "WARNING",
|
||||
"PIL": "WARNING",
|
||||
"alembic": "INFO",
|
||||
# uvicorn's own startup/shutdown/WebSocket lines are worth keeping; its ACCESS log is not —
|
||||
# it is one line per request with the query string in it, replaced by `log_requests` below.
|
||||
"uvicorn": "INFO",
|
||||
"uvicorn.error": "INFO",
|
||||
"uvicorn.access": "CRITICAL",
|
||||
}
|
||||
|
||||
|
||||
def resolve_level(name: str) -> int:
|
||||
"""Level name → number. Both ways in (the `LOG_LEVEL` env and the `log_level` admin knob) are
|
||||
already held to the `LogLevel` choices, so an unknown name here is a programming error."""
|
||||
value = logging.getLevelName(name)
|
||||
if not isinstance(value, int):
|
||||
raise ValueError(f"unknown log level {name!r}")
|
||||
return value
|
||||
|
||||
|
||||
def configure_logging(source: str) -> None:
|
||||
"""Configure logging for THIS process. Call once, first thing, in every entry point:
|
||||
`main.py` (api), `worker.main`, `downloads/runner.main`, `alembic/env.py`.
|
||||
|
||||
`source` names the process in every line. The level starts at `LOG_LEVEL` (the bootstrap default);
|
||||
the runtime admin knob goes through `set_level()` instead, so changing it needs no restart.
|
||||
|
||||
Our loggers deliberately keep `propagate=True` and own no handler: everything reaches the single
|
||||
root StreamHandler. That is what keeps `caplog` working in tests, and it means a new logger name
|
||||
is configured the moment it is created."""
|
||||
global _source
|
||||
_source = source
|
||||
formatter = settings.log_format
|
||||
logging.config.dictConfig(
|
||||
{
|
||||
"version": 1,
|
||||
# Never disable loggers created at import time (module-level `getLogger` calls run
|
||||
# before this function does).
|
||||
"disable_existing_loggers": False,
|
||||
"formatters": {
|
||||
"json": {"()": f"{__name__}.JsonFormatter"},
|
||||
"text": {"()": f"{__name__}.TextFormatter"},
|
||||
},
|
||||
"filters": {
|
||||
"context": {"()": f"{__name__}.ContextFilter"},
|
||||
"gate": {"()": f"{__name__}.LevelGate"},
|
||||
},
|
||||
"handlers": {
|
||||
"stdout": {
|
||||
"class": "logging.StreamHandler",
|
||||
"stream": sys.stdout,
|
||||
"formatter": formatter,
|
||||
"filters": ["context", "gate"],
|
||||
}
|
||||
},
|
||||
"root": {"level": resolve_level(settings.log_level), "handlers": ["stdout"]},
|
||||
"loggers": {name: {"level": lvl} for name, lvl in _THIRD_PARTY_LEVELS.items()},
|
||||
}
|
||||
)
|
||||
set_level(settings.log_level)
|
||||
|
||||
|
||||
def set_level(name: str) -> None:
|
||||
"""Change the running process's level (the `log_level` admin knob). Third-party loggers keep
|
||||
their pinned levels, so turning on DEBUG shows OUR detail, not SQLAlchemy's."""
|
||||
global _base_level
|
||||
_base_level = resolve_level(name)
|
||||
_apply_root_level(force=True)
|
||||
|
||||
|
||||
def current_level() -> str:
|
||||
"""The configured level — not the root logger's, which an active elevation may have lowered."""
|
||||
return logging.getLevelName(_base_level or logging.getLogger().getEffectiveLevel())
|
||||
|
||||
|
||||
# --- per-user elevation (E22 S5) -----------------------------------------------------------
|
||||
# The base level (`LOG_LEVEL`, then the admin knob). NOTSET until configure_logging runs — a unit
|
||||
# test, a script — so the gate lets everything through there, as it did before the gate existed.
|
||||
_base_level = logging.NOTSET
|
||||
# user id → (level number, expiry) for every elevation this process has seen and not yet expired.
|
||||
_elevations: dict[int, tuple[int, datetime]] = {}
|
||||
_elevations_lock = threading.Lock()
|
||||
_root_applied: int | None = None
|
||||
|
||||
|
||||
def _track_elevation(user_id: int, level: int | None, until: datetime | None) -> None:
|
||||
with _elevations_lock:
|
||||
if level is None or until is None:
|
||||
_elevations.pop(user_id, None)
|
||||
else:
|
||||
_elevations[user_id] = (level, until)
|
||||
active = bool(_elevations)
|
||||
if active or _root_applied != _base_level:
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def _apply_root_level(force: bool = False) -> None:
|
||||
"""Root level = the lowest of the base and every unexpired elevation (expired ones are pruned
|
||||
here). `setLevel` clears every logger's level cache, so it runs only when the number changes."""
|
||||
global _root_applied
|
||||
if _base_level == logging.NOTSET:
|
||||
return # not configured: the root keeps whatever the process set up
|
||||
now = datetime.now(timezone.utc)
|
||||
# The level is SET under the lock too, not just computed: two threads that each computed a
|
||||
# different answer would otherwise apply them in either order, and the older one could win —
|
||||
# the root back at the base while an elevation is still active.
|
||||
with _elevations_lock:
|
||||
for user_id in [u for u, (_, until) in _elevations.items() if until <= now]:
|
||||
del _elevations[user_id]
|
||||
lowest = min([_base_level, *(lvl for lvl, _ in _elevations.values())])
|
||||
if force or lowest != _root_applied:
|
||||
logging.getLogger().setLevel(lowest)
|
||||
_root_applied = lowest
|
||||
|
||||
|
||||
# --- erased accounts ------------------------------------------------------------------------
|
||||
# `auth.purge_user` deletes an account's `log_events` rows, but a line about that account can still
|
||||
# be on its way: the erasing request's own access line is logged AFTER the handler committed, a queued
|
||||
# row waits for the sink's writer, a background task logs later. Those rows are stored without the
|
||||
# account (`logsink` asks here). Remembered for an hour in this process — a line later than that has
|
||||
# no request left to come from, and the retention job is the backstop for any other process.
|
||||
ERASED_MEMORY_SECONDS = 3600.0
|
||||
_erased: dict[int, float] = {}
|
||||
_erased_lock = threading.Lock()
|
||||
# Held by whoever checks rows against `is_erased` AND writes them (`erasure_guard`). `forget_user`
|
||||
# takes it too, so an erasure waits for a write that already passed the check to commit — and the
|
||||
# purge's DELETE, which runs after `forget_user`, then removes those rows as well. Without it a row
|
||||
# checked a moment before the erasure could be inserted a moment after the DELETE.
|
||||
_erasure_guard = threading.Lock()
|
||||
|
||||
|
||||
def erasure_guard() -> threading.Lock:
|
||||
"""Hold this around "drop erased accounts from these rows, insert them, commit"."""
|
||||
return _erasure_guard
|
||||
|
||||
|
||||
def forget_user(user_id: int) -> None:
|
||||
"""Mark `user_id` erased: no line logged about it from now on is stored under its id. Call BEFORE
|
||||
deleting its rows (see `_erasure_guard`)."""
|
||||
now = time.monotonic()
|
||||
with _erasure_guard, _erased_lock:
|
||||
for stale in [u for u, at in _erased.items() if now - at > ERASED_MEMORY_SECONDS]:
|
||||
del _erased[stale]
|
||||
_erased[user_id] = now
|
||||
with _elevations_lock:
|
||||
_elevations.pop(user_id, None)
|
||||
# That pop can have removed the lowest elevation, and the root level is the lowest of the base
|
||||
# and every unexpired one. Recomputed here rather than left to whichever later request happens
|
||||
# to call `_track_elevation` — the invariant `_apply_root_level` documents holds at every pop.
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def is_erased(user_id: int | None) -> bool:
|
||||
if user_id is None:
|
||||
return False
|
||||
with _erased_lock:
|
||||
at = _erased.get(user_id)
|
||||
return at is not None and time.monotonic() - at <= ERASED_MEMORY_SECONDS
|
||||
|
||||
|
||||
def current_source() -> str:
|
||||
"""The process name `configure_logging` was given (api, worker, runner, alembic)."""
|
||||
return _source
|
||||
|
||||
|
||||
# --- request logging -----------------------------------------------------------------------
|
||||
# A client may supply its own request id so a browser log line and a server line join up (E22 S4).
|
||||
# Accepted only in this shape: it ends up in a log field, and an unbounded header would be a way to
|
||||
# write arbitrary text into the log.
|
||||
_REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,64}$")
|
||||
|
||||
# Paths whose success is logged at DEBUG rather than INFO: the container healthcheck runs every 15s
|
||||
# (5.7k lines/day saying nothing) and hashed static assets are served by the hundred per page load.
|
||||
# A FAILURE on either still logs at WARNING/ERROR — this lowers the floor, it does not hide.
|
||||
_QUIET_PREFIXES = ("/healthz", "/assets/", "/static/")
|
||||
|
||||
_request_log = logging.getLogger("siftlode.http")
|
||||
|
||||
|
||||
async def log_requests(request, call_next):
|
||||
"""The single access-log line, replacing uvicorn's.
|
||||
|
||||
What changes versus `uvicorn.access`: the ROUTE TEMPLATE instead of the concrete path (so
|
||||
`/api/videos/{video_id}` groups), never the query string (it carried `?account=`, and at TRACE
|
||||
the redacted query belongs in attrs, not in the URL), a request id echoed back in
|
||||
`X-Request-Id` (with the template in `X-Route`), the duration, and the account — read from `request.state`, which the auth
|
||||
dependency fills, because this middleware sits outside both the session middleware and the
|
||||
downstream task's context."""
|
||||
incoming = request.headers.get("x-request-id", "")
|
||||
rid = incoming if _REQUEST_ID_RE.match(incoming) else uuid.uuid4().hex[:16]
|
||||
token = _request_id.set(rid)
|
||||
scope = _RequestScope()
|
||||
scope_token = _scope.set(scope)
|
||||
request.state.request_id = rid
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
# The unhandled-exception path: FastAPI's ServerErrorMiddleware re-raises after its own
|
||||
# 500, so without this the ONE request that most needs a log line is the one with no
|
||||
# access-log entry at all.
|
||||
_request_log.exception(
|
||||
"request failed",
|
||||
extra={
|
||||
**event(
|
||||
"http.request",
|
||||
method=request.method,
|
||||
path=_route_template(request),
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 1),
|
||||
),
|
||||
**_request_owner(request, logging.ERROR),
|
||||
},
|
||||
)
|
||||
_scope.reset(scope_token)
|
||||
_request_id.reset(token)
|
||||
raise
|
||||
duration_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
status = response.status_code
|
||||
quiet = request.url.path.startswith(_QUIET_PREFIXES)
|
||||
if status >= 500:
|
||||
level = logging.ERROR
|
||||
elif status >= 400:
|
||||
level = logging.WARNING
|
||||
else:
|
||||
level = logging.DEBUG if quiet else logging.INFO
|
||||
attrs: dict[str, Any] = {
|
||||
"method": request.method,
|
||||
"path": _route_template(request),
|
||||
"status": status,
|
||||
"duration_ms": duration_ms,
|
||||
}
|
||||
# Teljes (TRACE) the user consented to: the query string too — redacted, and in attrs only, never
|
||||
# in the message.
|
||||
if request.url.query and _traced(scope):
|
||||
attrs["query"] = redact_query(request.url.query)
|
||||
_request_log.log(
|
||||
level,
|
||||
"%s %s %s",
|
||||
request.method,
|
||||
_route_template(request),
|
||||
status,
|
||||
extra={**event("http.request", **attrs), **_request_owner(request, level)},
|
||||
)
|
||||
response.headers["X-Request-Id"] = rid
|
||||
# The template, so the browser's copy of this line groups the same way (frontend lib/http.ts).
|
||||
# Only a MATCHED route's: a template never carries a path parameter's value (a share token stays
|
||||
# `{token}`), while the raw-path fallback of an unmatched request could — the browser then
|
||||
# templates the path itself.
|
||||
matched = _matched_route(request)
|
||||
if matched:
|
||||
response.headers["X-Route"] = matched
|
||||
_scope.reset(scope_token)
|
||||
_request_id.reset(token)
|
||||
return response
|
||||
|
||||
|
||||
def _traced(scope: _RequestScope) -> bool:
|
||||
"""Whether this request's query string is logged: only under its OWN user's Teljes consent. An
|
||||
admin setting the whole process to TRACE gets more detail, not every user's search terms — the
|
||||
privacy policy promises those are kept only for a user who agreed to it."""
|
||||
return scope.level is not None and scope.level <= TRACE
|
||||
|
||||
|
||||
# Anonymous request lines kept per client per minute. Signed-in traffic is not capped: it is a known
|
||||
# account and exactly what an admin needs to see. Anonymous traffic is where the volume can come from
|
||||
# nobody in particular — a scanner walking `/wp-login.php`, a bot on a share link — and each of its
|
||||
# 404s is a WARNING that would otherwise land in `log_events` without limit. The client key is an
|
||||
# in-memory counter key only; no address is ever written to a log line or a row.
|
||||
ANONYMOUS_LINES_PER_MINUTE = 60
|
||||
_anonymous_lines = RateLimiter(ANONYMOUS_LINES_PER_MINUTE, 60.0)
|
||||
|
||||
|
||||
def _request_owner(request, level: int) -> dict[str, Any]:
|
||||
"""`extra=` fields naming who the request belongs to, and whether its line may be stored. The
|
||||
account comes from `request.state` (filled by the auth dependency), so it lands in the line's
|
||||
top-level `user_id` — the correlation field — rather than in `attrs`."""
|
||||
user_id = getattr(request.state, "user_id", None)
|
||||
if user_id is not None:
|
||||
return {"user_id": user_id}
|
||||
if level < logging.INFO:
|
||||
return {} # a quiet line is never stored anyway — don't spend the client's allowance on it
|
||||
return {} if _anonymous_lines.allow(client_key(request)) else {PERSIST: False}
|
||||
|
||||
|
||||
def _matched_route(request) -> str | None:
|
||||
"""The matched route's template (`/api/videos/{video_id}`), or None when nothing matched."""
|
||||
return getattr(request.scope.get("route"), "path", None)
|
||||
|
||||
|
||||
def _route_template(request) -> str:
|
||||
"""The matched route's template, else the raw path for a 404 or a static file. Never the query
|
||||
string."""
|
||||
return _matched_route(request) or request.url.path
|
||||
@@ -0,0 +1,315 @@
|
||||
"""The `log_events` sink (E22 S3): the stored copy of the log the diagnostics console reads.
|
||||
|
||||
stdout stays the complete log (docker keeps 3×50 MB of it); this table keeps the part worth querying
|
||||
for `log_retention_days`:
|
||||
|
||||
* every WARNING and above,
|
||||
* the `http.request` line of every request at INFO or above (not the DEBUG-level health checks and
|
||||
static files),
|
||||
* every line of a request whose user raised their diagnostics level (E22 S5 — `logs.LevelGate`
|
||||
marks them), which is the detail they consented to have kept,
|
||||
* nothing a caller marked `extra={logs.PERSIST: False}` — how the request log caps anonymous
|
||||
traffic per client.
|
||||
|
||||
What the table must never do is slow down or break the code that logs. So a line is redacted and
|
||||
turned into a row on the CALLER's thread (the same `logs.record_fields` the stdout line is built
|
||||
from), put on a bounded queue without blocking, and written in batches by one background thread
|
||||
on its own connection — never inside a request's transaction. A full queue drops the row and
|
||||
counts it; a failed write drops the batch and says so on stderr. Both are reported rather than
|
||||
retried: a log that retries into a sick database is how a small outage becomes a big one.
|
||||
|
||||
Flood control is the second half. On 2026-09-16 a YouTube RSS outage made prod log 11k near-identical
|
||||
warnings in four hours. stdout keeps them all; the table keeps `REPEATS_PER_MINUTE` of each message
|
||||
template per minute and one `log.repeats_suppressed` row saying how many more there were.
|
||||
|
||||
Core layer: imports `logs`, `db`, `models`, `config` — nothing above them.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, insert
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.db import engine
|
||||
from app.models import LogEvent
|
||||
|
||||
QUEUE_MAX = 10_000
|
||||
BATCH_MAX = 500
|
||||
FLUSH_SECONDS = 1.0
|
||||
REPEATS_PER_MINUTE = 30
|
||||
REPEAT_WINDOW_SECONDS = 60.0
|
||||
# Distinct templates tracked per window. Past this many, a new template is stored without counting
|
||||
# — the limiter must not become the unbounded thing it exists to prevent.
|
||||
_REPEAT_KEYS_MAX = 2_000
|
||||
|
||||
# Column widths from models.LogEvent. A value past its column would fail the WHOLE batch it rides in.
|
||||
_LIMITS = {
|
||||
"level": 10,
|
||||
"source": 16,
|
||||
"logger": 128,
|
||||
"event": 64,
|
||||
"request_id": 64,
|
||||
"debug_session_id": 64,
|
||||
"version": 64,
|
||||
"msg": 8_000,
|
||||
"exc": 32_000,
|
||||
}
|
||||
|
||||
_log = logging.getLogger("siftlode.logsink")
|
||||
|
||||
|
||||
def should_store(record: logging.LogRecord) -> bool:
|
||||
"""Whether this record belongs in `log_events` (see the module docstring)."""
|
||||
if getattr(record, logs.PERSIST, True) is False:
|
||||
return False
|
||||
if record.levelno >= logging.WARNING or getattr(record, "elevated", False):
|
||||
return True
|
||||
return getattr(record, "event", None) == "http.request" and record.levelno >= logging.INFO
|
||||
|
||||
|
||||
def _without_nul(value: Any) -> Any:
|
||||
"""`value` with every NUL character replaced, in keys and nested values alike. Postgres refuses
|
||||
NUL in text and `\\u0000` in JSONB, and a refused row fails the WHOLE batch it rides in — every
|
||||
user's rows with it. Anything logged can carry one (`GET /assets/%00` logs the decoded path)."""
|
||||
if isinstance(value, str):
|
||||
return value.replace("\x00", "�")
|
||||
if isinstance(value, dict):
|
||||
return {_without_nul(k): _without_nul(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_without_nul(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def to_row(fields: dict[str, Any]) -> dict[str, Any]:
|
||||
"""`logs.record_fields` output → a `log_events` row: known columns only, text clipped to its
|
||||
column and free of NUL, attrs made JSON-safe (anything exotic becomes its `str`, as on stdout)."""
|
||||
row = {
|
||||
column: fields.get(column) for column in LogEvent.__table__.columns.keys() if column != "id"
|
||||
}
|
||||
for column, limit in _LIMITS.items():
|
||||
value = row.get(column)
|
||||
if isinstance(value, str):
|
||||
value = _without_nul(value)
|
||||
row[column] = value[: limit - 1] + "…" if len(value) > limit else value
|
||||
if row["attrs"] is not None:
|
||||
row["attrs"] = _without_nul(
|
||||
json.loads(json.dumps(row["attrs"], ensure_ascii=False, default=str))
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def without_erased(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""The rows, with the account taken off any whose account was erased meanwhile
|
||||
(`logs.forget_user`). Checked right before the INSERT, under `logs.erasure_guard()` held until
|
||||
the commit: a row built before the erasure can still be waiting for its batch."""
|
||||
for row in rows:
|
||||
if logs.is_erased(row.get("user_id")):
|
||||
row["user_id"] = None
|
||||
return rows
|
||||
|
||||
|
||||
def _notice(code: str, msg: str, **attrs: Any) -> dict[str, Any]:
|
||||
"""A row the sink writes about ITSELF. Built directly, not logged: logging it would route it back
|
||||
through this sink (and the writer thread's own records are deliberately ignored)."""
|
||||
return to_row(
|
||||
{
|
||||
"ts": datetime.now(timezone.utc),
|
||||
"level": "WARNING",
|
||||
"sev": logs.SEVERITY["WARNING"],
|
||||
"source": logs.current_source(),
|
||||
"logger": _log.name,
|
||||
"event": code,
|
||||
"msg": msg,
|
||||
"attrs": attrs,
|
||||
"version": settings.app_version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _Repeats:
|
||||
"""Per-template admission for one minute at a time. Used by the writer thread only (unlocked)."""
|
||||
|
||||
def __init__(self, per_window: int, window_seconds: float):
|
||||
self.per_window = per_window
|
||||
self.window = window_seconds
|
||||
self.started = time.monotonic()
|
||||
self.seen: dict[tuple[str, str, int], int] = {}
|
||||
|
||||
def admit(self, key: tuple[str, str, int] | None) -> bool:
|
||||
if key is None:
|
||||
return True
|
||||
count = self.seen.get(key)
|
||||
if count is None and len(self.seen) >= _REPEAT_KEYS_MAX:
|
||||
return True
|
||||
self.seen[key] = (count or 0) + 1
|
||||
return self.seen[key] <= self.per_window
|
||||
|
||||
def roll(self, now: float) -> list[dict[str, Any]]:
|
||||
"""Close the window if it is over: one notice per template that went past the limit."""
|
||||
if now - self.started < self.window:
|
||||
return []
|
||||
notices = [
|
||||
_notice(
|
||||
"log.repeats_suppressed",
|
||||
f"{count - self.per_window} more like this were not stored",
|
||||
logger=logger,
|
||||
template=logs.redact_text(template),
|
||||
suppressed=count - self.per_window,
|
||||
window_seconds=int(self.window),
|
||||
)
|
||||
for (logger, template, _level), count in self.seen.items()
|
||||
if count > self.per_window
|
||||
]
|
||||
self.seen.clear()
|
||||
self.started = now
|
||||
return notices
|
||||
|
||||
|
||||
class LogSink(logging.Handler):
|
||||
"""The handler plus its writer. `start()` runs the writer thread; a test drives `write_pending()`
|
||||
directly instead, on its own thread, with no timing involved."""
|
||||
|
||||
def __init__(self, bind: Engine = engine):
|
||||
super().__init__(level=logging.NOTSET)
|
||||
# Correlation fields are contextvars: capture them now, on the thread that logged.
|
||||
self.addFilter(logs.ContextFilter())
|
||||
self._bind = bind
|
||||
self._queue: queue.Queue[tuple[tuple[str, str, int] | None, dict[str, Any]]] = queue.Queue(
|
||||
QUEUE_MAX
|
||||
)
|
||||
self._dropped = 0
|
||||
self._dropped_lock = threading.Lock()
|
||||
self._repeats = _Repeats(REPEATS_PER_MINUTE, REPEAT_WINDOW_SECONDS)
|
||||
self._stopping = threading.Event()
|
||||
self._writer: threading.Thread | None = None
|
||||
|
||||
# --- caller side ---------------------------------------------------------------------------
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
if threading.current_thread() is self._writer:
|
||||
return # the writer's own trouble (a driver warning mid-insert) must not feed itself
|
||||
if not should_store(record):
|
||||
return
|
||||
try:
|
||||
row = to_row(logs.record_fields(record))
|
||||
except Exception: # noqa: BLE001 — a record we cannot turn into a row is reported, not raised
|
||||
self.handleError(record)
|
||||
return
|
||||
# Request lines are one per request by nature; only the repeated SAME message is a flood.
|
||||
key = (
|
||||
None
|
||||
if row["event"] == "http.request"
|
||||
else (record.name, str(record.msg), record.levelno)
|
||||
)
|
||||
try:
|
||||
self._queue.put_nowait((key, row))
|
||||
except queue.Full:
|
||||
with self._dropped_lock:
|
||||
self._dropped += 1
|
||||
|
||||
# --- writer side ---------------------------------------------------------------------------
|
||||
def write_pending(self, wait: float = 0.0) -> int:
|
||||
"""Write one batch: whatever is queued (up to BATCH_MAX), waiting up to `wait` seconds for the
|
||||
first item. Returns the number of rows written, notices included."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
item = self._queue.get(timeout=wait) if wait > 0 else self._queue.get_nowait()
|
||||
except queue.Empty:
|
||||
item = None
|
||||
while item is not None:
|
||||
key, row = item
|
||||
if self._repeats.admit(key):
|
||||
rows.append(row)
|
||||
if len(rows) >= BATCH_MAX:
|
||||
break
|
||||
try:
|
||||
item = self._queue.get_nowait()
|
||||
except queue.Empty:
|
||||
item = None
|
||||
rows.extend(self._repeats.roll(time.monotonic()))
|
||||
with self._dropped_lock:
|
||||
dropped, self._dropped = self._dropped, 0
|
||||
if dropped:
|
||||
rows.append(
|
||||
_notice("log.sink.dropped", f"{dropped} log events were dropped", dropped=dropped)
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
try:
|
||||
# The connection is checked out BEFORE the guard, and every request path takes them in
|
||||
# that order too (its session already holds a connection when it reaches the guard). The
|
||||
# other way round, this thread would hold a process-global lock while waiting up to
|
||||
# `pool_timeout` for a connection, and the requests that could return one are the ones
|
||||
# blocked on the lock. The guard still spans the check, the insert and the commit.
|
||||
with self._bind.connect() as conn, logs.erasure_guard(), conn.begin():
|
||||
conn.execute(insert(LogEvent), without_erased(rows))
|
||||
except Exception as exc: # noqa: BLE001 — the database is down or refused the batch
|
||||
print(
|
||||
f"log sink: could not store {len(rows)} log events: {logs.redact_text(str(exc))[:300]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
return len(rows)
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stopping.is_set():
|
||||
self.write_pending(wait=FLUSH_SECONDS)
|
||||
while self.write_pending(): # drain what arrived before the stop
|
||||
pass
|
||||
|
||||
def start(self) -> None:
|
||||
self._writer = threading.Thread(target=self._run, name="log-sink", daemon=True)
|
||||
self._writer.start()
|
||||
|
||||
def stop(self, timeout: float = 5.0) -> None:
|
||||
self._stopping.set()
|
||||
if self._writer is not None:
|
||||
self._writer.join(timeout)
|
||||
|
||||
|
||||
_sink: LogSink | None = None
|
||||
|
||||
|
||||
def start() -> None:
|
||||
"""Attach the sink to the root logger and start its writer. Call after `logs.configure_logging`
|
||||
(which replaces the root handlers) and once the schema exists. A no-op when `LOG_DB_SINK` is off
|
||||
or it is already running."""
|
||||
global _sink
|
||||
if _sink is not None or not settings.log_db_sink:
|
||||
return
|
||||
_sink = LogSink()
|
||||
# The same level gate as stdout: the root may be lowered for one user's elevated requests, and
|
||||
# everyone else's detail must not reach the table through that gap either.
|
||||
_sink.addFilter(logs.LevelGate())
|
||||
logging.getLogger().addHandler(_sink)
|
||||
_sink.start()
|
||||
# A short-lived process (the download child) exits without a lifespan: flush on the way out.
|
||||
atexit.register(stop)
|
||||
|
||||
|
||||
def stop() -> None:
|
||||
"""Detach the sink and write what is still queued."""
|
||||
global _sink
|
||||
if _sink is None:
|
||||
return
|
||||
logging.getLogger().removeHandler(_sink)
|
||||
_sink.stop()
|
||||
_sink = None
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Stage the deletion of rows older than `retention_days` (at least one day — stored logs always
|
||||
expire). The caller commits. Returns the number of rows removed."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, retention_days))
|
||||
result = db.execute(delete(LogEvent).where(LogEvent.ts < cutoff))
|
||||
return result.rowcount or 0
|
||||
@@ -175,7 +175,7 @@ MAIL_TEXT: dict[str, dict[str, str]] = {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Someone has opened up Siftlode for you — a quiet, self-hosted reader for your YouTube "
|
||||
"subscriptions. Every upload from the channels you follow lands in one feed you can "
|
||||
"subscriptions. Every upload from the channels you follow lands in one Feed you can "
|
||||
"actually filter and sort, with no algorithm deciding what you see, and no Shorts or "
|
||||
"livestreams unless you want them.\n\n"
|
||||
"{next_step}\n\n"
|
||||
@@ -186,7 +186,7 @@ MAIL_TEXT: dict[str, dict[str, str]] = {
|
||||
"Szia,\n\n"
|
||||
"Valaki megnyitotta előtted a Siftlode-ot — ez egy csendes, saját üzemeltetésű olvasó "
|
||||
"a YouTube-feliratkozásaidhoz. A követett csatornák minden feltöltése egyetlen "
|
||||
"hírfolyamba érkezik, amit tényleg tudsz szűrni és rendezni; nincs algoritmus, ami "
|
||||
"Feedbe érkezik, amit tényleg tudsz szűrni és rendezni; nincs algoritmus, ami "
|
||||
"eldönti, mit látsz, és nincs Shorts vagy élő adás, hacsak nem kéred.\n\n"
|
||||
"{next_step}\n\n"
|
||||
"Nem sürget semmi — az ajtó nyitva marad.\n\n"
|
||||
|
||||
+98
-15
@@ -1,38 +1,43 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import math
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
|
||||
# When started via uvicorn --log-config the "siftlode" logger is already configured
|
||||
# (see log_config.json). This block is a timestamped fallback for other entrypoints
|
||||
# (tests, scripts) so our logs are never silently dropped.
|
||||
_siftlode_logger = logging.getLogger("siftlode")
|
||||
if not _siftlode_logger.handlers:
|
||||
_handler = logging.StreamHandler(sys.stdout)
|
||||
_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s"))
|
||||
_siftlode_logger.addHandler(_handler)
|
||||
_siftlode_logger.setLevel(logging.INFO)
|
||||
_siftlode_logger.propagate = False
|
||||
# Logging is configured HERE, before the route modules are imported, so every logger they create at
|
||||
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
||||
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
||||
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||
|
||||
configure_logging("api")
|
||||
|
||||
log = logging.getLogger("siftlode.app")
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.exc import DataError
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from app import logsink
|
||||
from app import state
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.plex import stream as plex_stream
|
||||
from app.plex import subtitles as plex_subtitles
|
||||
from app.routes import (
|
||||
admin,
|
||||
audit as audit_routes,
|
||||
auth as auth_routes,
|
||||
channels,
|
||||
config as config_routes,
|
||||
diagnostics as diagnostics_routes,
|
||||
discover as discover_routes,
|
||||
downloads,
|
||||
feed,
|
||||
health,
|
||||
@@ -57,9 +62,21 @@ from app.scheduler import shutdown_scheduler, start_scheduler
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
log.info("Siftlode starting up")
|
||||
# The stored copy of the log (log_events). The entrypoint has already migrated the schema.
|
||||
logsink.start()
|
||||
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
||||
# the wizard URL so the operator can open it (the app runs in setup mode until they finish).
|
||||
with SessionLocal() as db:
|
||||
# The admin's runtime level (sysconfig `log_level`) wins over the LOG_LEVEL bootstrap
|
||||
# default, so a level chosen in the UI survives a restart without touching the environment.
|
||||
# Best-effort on purpose: a LOGGING knob must never be the thing that stops the app from
|
||||
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
||||
try:
|
||||
set_level(sysconfig.get_str(db, "log_level"))
|
||||
except Exception as exc: # noqa: BLE001 — any DB/config failure keeps the bootstrap level
|
||||
log.warning(
|
||||
"could not read the stored log level (%s); staying at %s", exc, current_level()
|
||||
)
|
||||
if not state.is_configured(db):
|
||||
token = state.rotate_setup_token(db)
|
||||
url = f"{settings.app_base}/setup?token={token}"
|
||||
@@ -76,6 +93,7 @@ async def lifespan(app: FastAPI):
|
||||
# alone doesn't buy that; not waiting does. The sweep only touches directories no live session
|
||||
# owns, so racing it against early playback is safe.
|
||||
sweep = asyncio.create_task(_sweep_hls_orphans())
|
||||
reaper = asyncio.create_task(_reap_hls_sessions())
|
||||
start_scheduler()
|
||||
try:
|
||||
yield
|
||||
@@ -85,10 +103,37 @@ async def lifespan(app: FastAPI):
|
||||
# pending ("Task was destroyed but it is pending!" — noise that reads like a bug during
|
||||
# triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own
|
||||
# bookkeeping honest.
|
||||
sweep.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await sweep
|
||||
for task in (sweep, reaper):
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
shutdown_scheduler()
|
||||
# Kills a running subtitle extraction: its worker would otherwise hold interpreter exit for
|
||||
# the rest of the read (see plex_subtitles.shutdown).
|
||||
plex_subtitles.shutdown()
|
||||
logsink.stop() # last: write what the shutdown itself just logged
|
||||
|
||||
|
||||
# How often idle HLS sessions are reaped (see plex/stream.py `reap_idle`).
|
||||
_HLS_REAP_EVERY_S = 60.0
|
||||
|
||||
|
||||
async def _reap_hls_sessions() -> None:
|
||||
"""The periodic reaper plex/stream.py's docstring promises. `reap_idle` used to run only inside
|
||||
the NEXT session request, so after the last playback of the day an abandoned session — its
|
||||
ffmpeg still working through the film, its directory a whole segmented copy of it — lived until
|
||||
somebody pressed play again or the process restarted.
|
||||
|
||||
A FINISHED session is reaped on the short rule here, not kept for the full idle window: an open
|
||||
player checkpoints every 10 s and that route touches the session, so a paused viewer is no longer
|
||||
invisible to this pass (see `reap_idle`). Keeping every finished session for ten minutes instead
|
||||
held one whole segmented copy of a film per item on the download volume (round 5)."""
|
||||
while True:
|
||||
await asyncio.sleep(_HLS_REAP_EVERY_S)
|
||||
try:
|
||||
await asyncio.to_thread(plex_stream.reap_idle)
|
||||
except Exception:
|
||||
log.exception("HLS session reap failed")
|
||||
|
||||
|
||||
async def _sweep_hls_orphans() -> None:
|
||||
@@ -107,6 +152,34 @@ async def _sweep_hls_orphans() -> None:
|
||||
|
||||
app = FastAPI(title=settings.app_name, lifespan=lifespan)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
"""FastAPI's own 422, in its own shape — except that it can be ENCODED. The JSON parser accepts
|
||||
`NaN`/`Infinity`, a float field refuses them, and the refusal echoes the value back: the default
|
||||
handler then failed to serialise its own error (JSON has no NaN) and the request became a 500
|
||||
with a traceback. A non-finite value is echoed as its name instead."""
|
||||
finite = {float: lambda x: x if math.isfinite(x) else str(x)}
|
||||
return JSONResponse(
|
||||
status_code=422, content={"detail": jsonable_encoder(exc.errors(), custom_encoder=finite)}
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(DataError)
|
||||
async def _nul_in_text(request: Request, exc: DataError) -> JSONResponse:
|
||||
"""A NUL byte in a string the request sent is a 422, on every route at once. Postgres cannot hold
|
||||
NUL in text, so the driver refuses to even send the query — and a `%00` in a query parameter or
|
||||
an escaped NUL in a JSON body (a playlist's `contains=`, a reorder's key list, a collection
|
||||
search) was a 500 with a traceback wherever it reached a comparison. Guarding each site did not
|
||||
hold: round 3 guarded the by-rating_key lookup and missed the rest. Any other DataError is left
|
||||
alone — re-raised, it stays the 500 it always was."""
|
||||
if "NUL (0x00)" not in str(exc.orig):
|
||||
raise exc
|
||||
return JSONResponse(
|
||||
status_code=422, content={"detail": {"code": "invalid_input", "reason": "nul"}}
|
||||
)
|
||||
|
||||
|
||||
app.add_middleware(
|
||||
SessionMiddleware,
|
||||
secret_key=settings.secret_key,
|
||||
@@ -182,13 +255,23 @@ async def security_headers(request, call_next):
|
||||
return response
|
||||
|
||||
|
||||
# The access log (app/logs.py): request id, route template, status, duration — one line, never a
|
||||
# query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole
|
||||
# request, including what the three above do. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# which is why the account comes from `request.state.user_id` (set by the auth dependency, the one
|
||||
# place a session is actually resolved into an account) rather than from `request.session`.
|
||||
app.middleware("http")(log_requests)
|
||||
|
||||
|
||||
app.include_router(health.router)
|
||||
app.include_router(auth_routes.router)
|
||||
app.include_router(sync.router)
|
||||
app.include_router(tags.router)
|
||||
app.include_router(feed.router)
|
||||
app.include_router(discover_routes.router)
|
||||
app.include_router(search_routes.router)
|
||||
app.include_router(me.router)
|
||||
app.include_router(diagnostics_routes.router)
|
||||
app.include_router(notifications.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(channels.router)
|
||||
|
||||
@@ -3,6 +3,7 @@ from datetime import date, datetime
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
Computed,
|
||||
Date,
|
||||
DateTime,
|
||||
@@ -11,6 +12,7 @@ from sqlalchemy import (
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
SmallInteger,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
@@ -23,6 +25,11 @@ from app.db import Base
|
||||
from app.utils import canonical_email
|
||||
|
||||
|
||||
# The resume-position columns (VideoState / PlexState.position_seconds) are int4. The checkpoint
|
||||
# routes bound their bodies by this, so an out-of-range position is a 422 rather than a 500.
|
||||
POSITION_MAX_S = 2**31 - 1
|
||||
|
||||
|
||||
class TimestampMixin:
|
||||
"""Adds a server-defaulted `created_at`. Models that index created_at or order it
|
||||
differently (QuotaEvent, Notification, Message) keep their own column on purpose."""
|
||||
@@ -96,6 +103,16 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
# current_user rejects any cookie whose epoch is stale, so a stolen/copied cookie dies on those
|
||||
# events (client-side signed cookies otherwise stay valid until they expire).
|
||||
session_epoch: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# When this account last signed in (Google or password), stamped by the one sign-in path,
|
||||
# routes/auth._establish_session. NULL = not since the column existed (0085; no backfill is
|
||||
# possible). The shared demo account is never stamped — its "last login" would mean nobody.
|
||||
last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The diagnostics level this user consented to (E22 S5) and when it lapses back to the base
|
||||
# level. Both NULL = base. Written only by `PUT /api/me/diagnostics`, which computes the expiry
|
||||
# itself; read through `app.diagnostics.effective_level`, never directly (an expired level is
|
||||
# not cleared eagerly — it simply stops counting).
|
||||
diagnostics_level: Mapped[str | None] = mapped_column(String(8))
|
||||
diagnostics_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
avatar_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
role: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
@@ -422,6 +439,7 @@ class VideoState(Base, UpdatedAtMixin):
|
||||
# status: a partially-watched video keeps status "new" but a non-zero position, so
|
||||
# it can render a progress bar and match the "in progress" feed filter. 0 = not
|
||||
# started / finished (trivially-early and near-finished positions are not stored).
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(
|
||||
Integer, default=0, server_default="0", nullable=False
|
||||
)
|
||||
@@ -532,6 +550,37 @@ class AuditLog(Base):
|
||||
)
|
||||
|
||||
|
||||
class LogEvent(Base):
|
||||
"""One stored log event (E22 S3) — the `log_events` table the diagnostics console reads. Rows are
|
||||
written by app/logsink.py's background writer, never inside a request's transaction, already
|
||||
redacted (the same `logs.record_fields` the stdout line is built from).
|
||||
|
||||
`user_id` has NO foreign key on purpose: the writer runs behind the event, so a line about an
|
||||
account can arrive after that account was purged, and a constraint would make it fail the whole
|
||||
batch it travels in. `auth.purge_user` deletes an account's rows explicitly instead, after
|
||||
`logs.forget_user` makes every later write in this process store such a line without the account;
|
||||
the retention job (`log_retention_days`) bounds whatever another process writes late."""
|
||||
|
||||
__tablename__ = "log_events"
|
||||
__table_args__ = (Index("ix_log_events_user_ts", "user_id", "ts"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
ts: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
level: Mapped[str] = mapped_column(String(10))
|
||||
sev: Mapped[int] = mapped_column(SmallInteger)
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
logger: Mapped[str] = mapped_column(String(128))
|
||||
event: Mapped[str | None] = mapped_column(String(64))
|
||||
msg: Mapped[str] = mapped_column(Text)
|
||||
attrs: Mapped[dict | None] = mapped_column(JSONB)
|
||||
exc: Mapped[str | None] = mapped_column(Text)
|
||||
request_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[int | None] = mapped_column(Integer)
|
||||
debug_session_id: Mapped[str | None] = mapped_column(String(64))
|
||||
job_id: Mapped[int | None] = mapped_column(Integer)
|
||||
version: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class AppState(Base):
|
||||
"""Single-row global app state (admin-controlled)."""
|
||||
|
||||
@@ -864,6 +913,8 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
error_code: Mapped[str | None] = mapped_column(
|
||||
String(24)
|
||||
) # category, see downloads.errors.classify
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
@@ -1406,3 +1457,52 @@ class PlexPlaylistItem(Base):
|
||||
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||
|
||||
|
||||
class ComfortPin(Base, TimestampMixin):
|
||||
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X" (E24).
|
||||
|
||||
Deliberately NOT a boolean on `VideoState`/`PlexState`, and NOT a polymorphic `(kind, ref)` string
|
||||
pair. A flag cannot ORDER a shelf and cannot hold a channel or a show — and comfort media is
|
||||
usually "put on X", a series or a channel, not one particular video. A string ref would order and
|
||||
hold anything, but it would also lose the foreign key: a Plex item dropped from the library, or an
|
||||
unsubscribed channel, would leave a dead tile on the one surface whose whole job is to be
|
||||
reliable. So instead: one nullable CASCADE-ing FK per pinnable kind, and a check constraint that
|
||||
exactly one of them is set.
|
||||
|
||||
That constraint is the house rule applied to a five-way union — the invariant lives in the schema,
|
||||
not in a comment every insert site has to remember. `kind`/`ref` below derive FROM the columns, so
|
||||
there is no second vocabulary to keep in sync.
|
||||
|
||||
NULLs are DISTINCT in a Postgres unique index, so the five per-kind unique constraints each stop a
|
||||
double-pin of the same thing without colliding across kinds."""
|
||||
|
||||
__tablename__ = "comfort_pins"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"num_nonnulls(video_id, channel_id, plex_item_id, plex_show_id, playlist_id) = 1",
|
||||
name="ck_comfort_pin_one_ref",
|
||||
),
|
||||
UniqueConstraint("user_id", "video_id", name="uq_comfort_pin_video"),
|
||||
UniqueConstraint("user_id", "channel_id", name="uq_comfort_pin_channel"),
|
||||
UniqueConstraint("user_id", "plex_item_id", name="uq_comfort_pin_plex_item"),
|
||||
UniqueConstraint("user_id", "plex_show_id", name="uq_comfort_pin_plex_show"),
|
||||
UniqueConstraint("user_id", "playlist_id", name="uq_comfort_pin_playlist"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
video_id: Mapped[str | None] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str | None] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||
plex_item_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_items.id", ondelete="CASCADE")
|
||||
)
|
||||
plex_show_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_shows.id", ondelete="CASCADE")
|
||||
)
|
||||
playlist_id: Mapped[int | None] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
|
||||
|
||||
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
|
||||
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||
|
||||
@@ -197,7 +197,12 @@ class PlexClient:
|
||||
)
|
||||
|
||||
def unscrobble(self, rating_key: str) -> None:
|
||||
"""Mark the item *unwatched* on the token's Plex account (clears viewCount)."""
|
||||
"""Mark the item *unwatched* on the token's Plex account.
|
||||
|
||||
Clears the view count AND the resume point (`viewOffset`) — measured against the live server
|
||||
on 2026-09-09, from both a watched+offset state and an offset-only one. This one call is the
|
||||
whole un-mark; `/:/timeline?time=0` and `/:/progress?time=0` were both tried there and do
|
||||
NOT clear an offset, so do not reach for them."""
|
||||
self._write(
|
||||
"GET",
|
||||
f"/:/unscrobble?identifier={self._LIBRARY_IDENTIFIER}"
|
||||
|
||||
@@ -11,5 +11,11 @@ _M = TypeVar("_M")
|
||||
def by_rating_key(db: Session, model: type[_M], rating_key: str) -> _M | None:
|
||||
"""The mirrored row for a Plex rating_key, or None. `model` is any of the rating_key-keyed Plex
|
||||
tables (PlexItem / PlexShow / PlexSeason / PlexCollection); the return type is that model, so call
|
||||
sites keep their `Model | None` static type."""
|
||||
sites keep their `Model | None` static type.
|
||||
|
||||
A key with a NUL in it (`%00` in a path segment) names no row — a 404, like any unknown key.
|
||||
Postgres refuses NUL in text, so asking would be a DataError; the app-wide handler turns the
|
||||
places that DO ask (lists and filters compared to rating_key) into a 422 (see main.py)."""
|
||||
if "\x00" in str(rating_key):
|
||||
return None
|
||||
return db.execute(select(model).where(model.rating_key == str(rating_key))).scalars().first()
|
||||
|
||||
@@ -8,11 +8,28 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import and_, case, func, or_, select, type_coerce
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import PlexItem
|
||||
from app.models import PlexItem, PlexLibrary
|
||||
|
||||
TS_CONFIG = "public.unaccent_simple"
|
||||
|
||||
# The ONE Plex sort vocabulary, shared by every surface that offers the Sort control (the browse grid
|
||||
# and the collection info page) and mirrored by the frontend's `plexSort.ts`. It lives here rather than
|
||||
# inside a router because the two surfaces order DIFFERENT things — browse orders a union subquery in
|
||||
# SQL, a collection orders its already-loaded member list in Python — and a key that one of them cannot
|
||||
# honour would put an option in the UI that silently does nothing. Tests pin both implementations to
|
||||
# this tuple, so adding a key here fails the gate until every surface can order by it.
|
||||
PLEX_SORT_KEYS: tuple[str, ...] = (
|
||||
"added",
|
||||
"lastwatched",
|
||||
"release",
|
||||
"year",
|
||||
"rating",
|
||||
"duration",
|
||||
"title",
|
||||
)
|
||||
|
||||
# --- Language-aware column pickers (E5 S6) ----------------------------------
|
||||
#
|
||||
# The HU-overlay twin (E5 S5) stores Hungarian metadata in `*_hu` columns on the SAME EN row. When
|
||||
@@ -342,3 +359,26 @@ def apply_show_state_filter(query, tot, wat, inp, show: str):
|
||||
if show == "unwatched":
|
||||
return query.where(wat == 0, inp == 0)
|
||||
return query
|
||||
|
||||
|
||||
def content_libs(db: Session) -> tuple[list[int], list[int]]:
|
||||
"""The BROWSABLE content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
|
||||
source of the browse scope. Excludes HU overlay libraries (`overlay_of` set): those are never
|
||||
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows (E5 S5).
|
||||
|
||||
Lives here, in the query layer, rather than in `routes/plex/_shared.py` where it started: E24's
|
||||
Home surface needs the same scope, and a domain package importing from `routes/` would invert the
|
||||
one-way dependency CLAUDE.md draws. It never touched request state, so nothing was lost by the
|
||||
move."""
|
||||
libs = (
|
||||
db.execute(
|
||||
select(PlexLibrary).where(
|
||||
PlexLibrary.enabled.is_(True), PlexLibrary.overlay_of.is_(None)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
movie_lib_ids = [lb.id for lb in libs if lb.kind == "movie"]
|
||||
show_lib_ids = [lb.id for lb in libs if lb.kind == "show"]
|
||||
return movie_lib_ids, show_lib_ids
|
||||
|
||||
@@ -3,11 +3,25 @@
|
||||
Drives the show/season grid badges and the Resume / Play-from-start / mark-all buttons.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.models import PlexItem, PlexState
|
||||
|
||||
# Sort sentinel for a rail row whose timestamp is missing, so `key=lambda r: r[0] or NO_TS` reads the
|
||||
# same at every rail that merges two type-homogeneous lists in Python.
|
||||
#
|
||||
# It is NOT a crash fix, whatever the E24 commit that introduced it says. The `(ts is not None, ts)`
|
||||
# key it replaced cannot raise: tuple comparison finds the first UNEQUAL element via `==`, so two
|
||||
# `(False, None)` keys compare equal and `None < None` is never reached — verified by execution
|
||||
# during the E24 epic-end review, over every permutation of null/non-null rows. What the sentinel
|
||||
# buys is one idiom instead of two, and a key that says "missing sorts last" outright rather than
|
||||
# encoding it in a boolean flag. A BARE `r[0]` would genuinely raise, which is the mutation the
|
||||
# rails' tests do guard against.
|
||||
NO_TS = datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def show_status(total: int, watched: int, inprog: int) -> str:
|
||||
"""Roll a show's episodes up into a single watch state: fully watched, partially started, or new."""
|
||||
|
||||
+208
-18
@@ -21,10 +21,12 @@ CPU-only prod host; only P3's full transcode is CPU-heavy.
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session as DbSession
|
||||
@@ -55,6 +57,10 @@ _SEG_SECONDS = 6
|
||||
# worse than the old zero-buffer behaviour).
|
||||
_WARMUP_SEGMENTS = 3
|
||||
_WARMUP_SEG_TIMEOUT_S = 8.0
|
||||
# The HLS variant ffmpeg gives the VIDEO rendition of a multi-audio session. `-var_stream_map` names
|
||||
# each audio rendition (`name:a0`, `a1`, …) but leaves the video one unnamed, so ffmpeg falls back to
|
||||
# its index for that one. See `segment_name` for why this is a constant and not a comment.
|
||||
_VIDEO_VARIANT = "0"
|
||||
_SESSION_IDLE_S = 600 # reap a session with no access for this long
|
||||
# A session directory is named `{rating_key}_{int(start_s)}_{tag}_{aoff:+.2f}` (see start_session);
|
||||
# before multi-audio it was just `{rating_key}_{int(start_s)}`. `sweep_orphans` deletes ONLY names
|
||||
@@ -94,11 +100,19 @@ class HlsSession:
|
||||
directory: Path,
|
||||
proc: subprocess.Popen,
|
||||
start_s: float,
|
||||
entry: str,
|
||||
variants: "Sequence[str]",
|
||||
transcode: bool = False,
|
||||
hw: bool = False,
|
||||
tonemap: bool = False,
|
||||
):
|
||||
# Unguessable per-session id, carried in the URLs the client is handed. Sessions are keyed
|
||||
# by rating_key alone, so a seek-restart or a SECOND viewer of the same item replaces the
|
||||
# registered one — and the old viewer's next request for `seg_12.ts`, resolved by rating_key,
|
||||
# was answered 200 with the NEW session's twelfth segment, produced from a different offset
|
||||
# under `-copyts`. Wrong bytes, not an error, so the client's reclaim never fired on it
|
||||
# (deep-review round 5). A request naming a session that is no longer registered is a 404,
|
||||
# which the player recovers from by restarting at the playhead.
|
||||
self.id = secrets.token_urlsafe(9)
|
||||
self.key = key
|
||||
self.dir = directory
|
||||
self.proc = proc
|
||||
@@ -113,7 +127,13 @@ class HlsSession:
|
||||
# subtitle cue-shift both key off the true content position — no drift. Provisional = X until
|
||||
# the first segment is probed.
|
||||
self.media_start_s = start_s
|
||||
self.entry = entry # the playlist filename hls.js should load (master.m3u8 with subs, else index.m3u8)
|
||||
# The rendition names this session's ffmpeg was told to write — () for a single-audio
|
||||
# session. Held on the session because it is what `artifact_may_arrive` needs to tell a name
|
||||
# this session CAN produce from one belonging to the other argv branch (see it).
|
||||
self.variants = tuple(variants)
|
||||
# The playlist filename hls.js should load: the master of a multi-audio session, else the
|
||||
# single index. Derived, not passed, so it cannot disagree with the argv.
|
||||
self.entry = _MASTER_PL if self.variants else _INDEX_PL
|
||||
self.last_access = time.time()
|
||||
|
||||
|
||||
@@ -271,6 +291,30 @@ def _hwaccel_in_opts(hw: bool, tonemap: bool = False) -> list[str]:
|
||||
return ["-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", "-vaapi_device", _VAAPI_DEVICE]
|
||||
|
||||
|
||||
# Audio is ALWAYS re-encoded on the HLS path: the source is typically AC3/DTS 5.1, which no browser
|
||||
# plays, so it is downmixed to stereo AAC. `-ac 2` uses ffmpeg's standard downmix matrix; nothing is
|
||||
# normalised, which is why a film mastered for a cinema arrives with whispered dialogue and deafening
|
||||
# action (bug hunt 2026-09, item 14).
|
||||
_AUDIO_ENC = ["-c:a", "aac", "-ac", "2", "-b:a", "192k"]
|
||||
# Optional loudness normalisation (admin knob `plex_audio_loudnorm`, default OFF). EBU R128 single-pass
|
||||
# `loudnorm` — two-pass is impossible here, since a seek-restart session must start emitting segments
|
||||
# immediately and cannot analyse the file first.
|
||||
#
|
||||
# The downmix is FORCED FIRST (`aformat=channel_layouts=stereo`) rather than left to `-ac 2`: measuring
|
||||
# 5.1 and then folding it down would normalise something other than what the viewer hears. `loudnorm`
|
||||
# runs its own resampler at 192 kHz internally, so the chain resamples back to 48 kHz for the AAC
|
||||
# encoder. Measured on a real 5.1 AC3 film (Westworld S1E6, 60 s from 15:00): I -27.6 → -16.6 LUFS,
|
||||
# LRA 13.3 → 9.8 LU — the quiet/loud gap narrows, and everything gets ~11 dB louder, so expect to turn
|
||||
# the volume down after switching this on. The filter's 3 s lookahead does NOT desync A/V: measured
|
||||
# under this pipeline's own `-copyts`, the audio start PTS is identical with and without it.
|
||||
_LOUDNORM_AF = "aformat=channel_layouts=stereo,loudnorm=I=-16:TP=-1.5:LRA=11,aresample=48000"
|
||||
|
||||
|
||||
def _aenc_opts(loudnorm: bool) -> list[str]:
|
||||
"""Audio encoder args for one output, with the normalisation filter when the admin enabled it."""
|
||||
return [*_AUDIO_ENC, *(["-af", _LOUDNORM_AF] if loudnorm else [])]
|
||||
|
||||
|
||||
def _venc_opts(
|
||||
transcode: bool,
|
||||
hw: bool,
|
||||
@@ -360,7 +404,7 @@ def _select_encode(db: DbSession, item: PlexItem, transcode: bool, src: Path) ->
|
||||
source that has an OpenCL runtime, while a 10-bit SDR source drops to software."""
|
||||
if not transcode:
|
||||
return (False, False)
|
||||
mode = (sysconfig.get_str(db, "plex_transcode_hw") or "auto").lower()
|
||||
mode = sysconfig.get_str(db, "plex_transcode_hw")
|
||||
if mode == "off" or not _render_node_present():
|
||||
return (False, False)
|
||||
if mode == "on":
|
||||
@@ -446,6 +490,7 @@ def _ffmpeg_cmd(
|
||||
preset: str = "veryfast",
|
||||
crf: int = 20,
|
||||
max_height: int = 0,
|
||||
loudnorm: bool = False,
|
||||
) -> list[str]:
|
||||
# For a transcode session the video is re-encoded (VAAPI or libx264) instead of stream-copied; the
|
||||
# hwaccel decode options go before -i, the encoder args replace `-c:v copy` below. Everything else is
|
||||
@@ -469,11 +514,15 @@ def _ffmpeg_cmd(
|
||||
if audio_langs:
|
||||
# MULTI-RENDITION: map EVERY audio track as an alternate HLS rendition in one group, so hls.js
|
||||
# switches audio CLIENT-SIDE on the same timeline (no session restart, no drift) — the premium
|
||||
# audio-switch path. Produces master.m3u8 + stream_%v.m3u8 (v0=video, v1..=audio) + seg_%v_%d.ts.
|
||||
# audio-switch path. Produces master.m3u8 + one stream_%v.m3u8 and seg_%v_*.ts per variant,
|
||||
# where %v is the video rendition's index ("0") and each audio rendition's `name:` (a0, a1,
|
||||
# …). Those names are spelled by `segment_name` / `variant_playlist_name` below, which the
|
||||
# guard and the warm-up read back — see them for the measurement behind the ordering.
|
||||
args += ["-map", "0:v:0"]
|
||||
for i in range(len(audio_langs)):
|
||||
args += ["-map", f"{ai}:a:{i}?"]
|
||||
args += ["-sn", *venc, "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
|
||||
# One `-af` covers EVERY audio output of this file, so each rendition is normalised.
|
||||
args += ["-sn", *venc, *_aenc_opts(loudnorm)]
|
||||
var = ["v:0,agroup:aud"]
|
||||
for i, lang in enumerate(audio_langs):
|
||||
entry = f"a:{i},agroup:aud,name:a{i}"
|
||||
@@ -482,23 +531,25 @@ def _ffmpeg_cmd(
|
||||
if i == 0:
|
||||
entry += ",default:yes"
|
||||
var.append(entry)
|
||||
# The name patterns come from the vocabulary functions, with ffmpeg's own placeholders passed
|
||||
# through them — so the producer and the parser cannot spell the layout differently (round 7).
|
||||
args += _HLS_TAIL + [
|
||||
"-master_pl_name",
|
||||
"master.m3u8",
|
||||
_MASTER_PL,
|
||||
"-var_stream_map",
|
||||
" ".join(var),
|
||||
"-hls_segment_filename",
|
||||
str(out_dir / "seg_%v_%d.ts"),
|
||||
str(out_dir / "stream_%v.m3u8"),
|
||||
str(out_dir / segment_name("%d", "%v")),
|
||||
str(out_dir / variant_playlist_name("%v")),
|
||||
]
|
||||
else:
|
||||
ao = audio_ord if audio_ord is not None else 0
|
||||
args += ["-map", "0:v:0", "-map", f"{ai}:a:{ao}?", "-sn"]
|
||||
args += [*venc, "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
|
||||
args += [*venc, *_aenc_opts(loudnorm)]
|
||||
args += _HLS_TAIL + [
|
||||
"-hls_segment_filename",
|
||||
str(out_dir / "seg_%d.ts"),
|
||||
str(out_dir / "index.m3u8"),
|
||||
str(out_dir / segment_name("%d")),
|
||||
str(out_dir / _INDEX_PL),
|
||||
]
|
||||
return args
|
||||
|
||||
@@ -579,16 +630,26 @@ def start_session(
|
||||
src = paths.local_media_path(db, item.file_path)
|
||||
if src is None:
|
||||
return None
|
||||
# Read once, here: the knob has to be identical for the directory name and the ffmpeg argv, and a
|
||||
# session respawn (the HW->SW fallback) must not pick up a mid-playback config change.
|
||||
loudnorm = sysconfig.get_bool(db, "plex_audio_loudnorm")
|
||||
key = item.rating_key
|
||||
start_s = max(0.0, float(start_s))
|
||||
audio_langs = _probe_audio_langs(src) if multi else []
|
||||
is_multi = len(audio_langs) > 1 # only worth a master playlist when there really are ≥2 tracks
|
||||
tag = "multi" if is_multi else str(audio_ord)
|
||||
# The rendition names the argv below will produce — one source for the argv, the warm-up's
|
||||
# segment names, the entry playlist and the artifact guard.
|
||||
variants = session_variants(audio_langs if is_multi else None)
|
||||
# Loudness normalisation changes the AUDIO THE SEGMENTS CONTAIN, so it belongs in the directory
|
||||
# name beside the other parameters that do (offset, audio track, A/V offset): a session left over
|
||||
# from before the admin flipped the knob must not be mistaken for one that honours it. Appended to
|
||||
# the tag with a DASH — `_SESSION_DIR_RE` treats the tag as one underscore-free field, and that
|
||||
# regex is the only thing keeping the orphan sweep off a shared scratch path (see its comment).
|
||||
tag = ("multi" if is_multi else str(audio_ord)) + ("-ln" if loudnorm else "")
|
||||
directory = _HLS_ROOT / f"{key}_{int(start_s)}_{tag}_{aoff:+.2f}"
|
||||
|
||||
def _seg(i: int) -> Path:
|
||||
# multi-audio: the video rendition is variant v0 → seg_{i}_0.ts; single-audio → seg_{i}.ts
|
||||
return directory / (f"seg_{i}_0.ts" if is_multi else f"seg_{i}.ts")
|
||||
return directory / segment_name(i, _VIDEO_VARIANT if variants else None)
|
||||
|
||||
def _spawn(
|
||||
hw: bool, tonemap: bool = False, *, replace_only: "HlsSession | None" = None
|
||||
@@ -603,7 +664,7 @@ def start_session(
|
||||
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
||||
preset, crf, max_height = "veryfast", 20, 0
|
||||
if transcode and not hw:
|
||||
preset = sysconfig.get_str(db, "plex_transcode_preset") or "veryfast"
|
||||
preset = sysconfig.get_str(db, "plex_transcode_preset")
|
||||
crf = sysconfig.get_int(db, "plex_transcode_crf")
|
||||
max_height = sysconfig.get_int(db, "plex_transcode_max_height")
|
||||
# Nice the SOFTWARE transcode child (not the HW path, and not remux) so a CPU-bound re-encode
|
||||
@@ -632,6 +693,7 @@ def start_session(
|
||||
preset=preset,
|
||||
crf=crf,
|
||||
max_height=max_height,
|
||||
loudnorm=loudnorm,
|
||||
),
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
@@ -642,7 +704,7 @@ def start_session(
|
||||
directory,
|
||||
proc,
|
||||
start_s,
|
||||
"master.m3u8" if is_multi else "index.m3u8",
|
||||
variants,
|
||||
transcode=transcode,
|
||||
hw=hw,
|
||||
tonemap=tonemap,
|
||||
@@ -714,6 +776,101 @@ def current_session(key: str) -> HlsSession | None:
|
||||
return s
|
||||
|
||||
|
||||
# The names ffmpeg writes into a session directory, in ONE place — and "one place" means the argv
|
||||
# BUILDS its patterns from these same functions (`segment_name("%d", "%v")`), not that a comment
|
||||
# says so. Getting this wrong is invisible in both directions: a name we failed to anticipate used
|
||||
# to fall back to "wait and see" (round 6's HIGH), and now that the rule is positive it falls back
|
||||
# to a 404 instead — so the producer and the parser have to be literally the same spelling.
|
||||
#
|
||||
# MEASURED (ffmpeg 7.1.5 in the API container — the binary that actually writes these files — and
|
||||
# re-checked on 8.1.1): `-hls_segment_filename seg_%v_%d.ts` substitutes `%v` with the VARIANT and
|
||||
# `%d` with the SEQUENCE — variant FIRST. A dual-audio session writes seg_0_0.ts… (video, the
|
||||
# unnamed variant) plus seg_a0_0.ts… / seg_a1_0.ts… (the `name:aN` audio renditions), and the
|
||||
# playlists beside them are master.m3u8 + stream_0/a0/a1.m3u8. Single-audio uses `seg_%d.ts` +
|
||||
# index.m3u8, with no variant field at all.
|
||||
#
|
||||
# The two argv branches are DISJOINT vocabularies, which is why the guard below is keyed to the
|
||||
# SESSION and not to HLS in general: a single-audio session can never write `master.m3u8`,
|
||||
# `stream_*.m3u8` or any `seg_<variant>_<n>.ts`, and a multi-audio one can never write `index.m3u8`
|
||||
# or `seg_<n>.ts`. Accepting the union meant a hopeless name of the other shape was waited out for
|
||||
# the full playlist/segment timeout — the threadpool parking this guard exists to end (round 7).
|
||||
_MASTER_PL = "master.m3u8"
|
||||
_INDEX_PL = "index.m3u8"
|
||||
_SEG_NAME_RE = re.compile(r"^seg_(?:([0-9a-z]{1,4})_)?(\d{1,9})\.ts$")
|
||||
|
||||
|
||||
def session_variants(audio_langs: "Sequence[str | None] | None") -> tuple[str, ...]:
|
||||
"""The `-var_stream_map` rendition names a session will write, or () for a single-audio one.
|
||||
|
||||
The video rendition is left unnamed in the map, so ffmpeg falls back to its index
|
||||
(`_VIDEO_VARIANT`); each audio rendition is named `aN` in track order."""
|
||||
if not audio_langs:
|
||||
return ()
|
||||
return (_VIDEO_VARIANT, *(f"a{i}" for i in range(len(audio_langs))))
|
||||
|
||||
|
||||
def segment_name(seq: "int | str", variant: "str | None" = None) -> str:
|
||||
"""The file ffmpeg writes for the `seq`-th segment of `variant`.
|
||||
|
||||
`variant=None` is a single-audio session, whose argv carries no `%v` field at all. `seq` and
|
||||
`variant` are typed loosely because `_ffmpeg_cmd` builds its own `-hls_segment_filename` pattern
|
||||
through this function, passing ffmpeg's `"%d"` / `"%v"` placeholders — that is what keeps the
|
||||
producer and the parser from drifting, rather than a comment asking the next editor not to."""
|
||||
return f"seg_{variant}_{seq}.ts" if variant else f"seg_{seq}.ts"
|
||||
|
||||
|
||||
def variant_playlist_name(variant: str) -> str:
|
||||
"""The sub-playlist ffmpeg writes for one rendition of a multi-audio session."""
|
||||
return f"stream_{variant}.m3u8"
|
||||
|
||||
|
||||
def playlist_names(variants: "Sequence[str]") -> frozenset[str]:
|
||||
"""Every playlist name this session's argv can write — the master plus one per rendition, or
|
||||
just `index.m3u8` for a single-audio session."""
|
||||
if not variants:
|
||||
return frozenset({_INDEX_PL})
|
||||
return frozenset({_MASTER_PL, *(variant_playlist_name(v) for v in variants)})
|
||||
|
||||
|
||||
def artifact_may_arrive(s: "HlsSession", filename: str) -> bool:
|
||||
"""Whether waiting for `filename` can possibly end in a file OF THIS SESSION.
|
||||
|
||||
Positive rule, not a blocklist: a session produces segments and playlists and NOTHING else, so
|
||||
any other name is refused at once. ffmpeg also produces segments strictly in order, so a segment
|
||||
whose PREDECESSOR does not exist yet is not the next one — it is beyond the generated region,
|
||||
and no amount of waiting will conjure it. Playlists are the one thing that must still be waited
|
||||
for: they are rewritten continuously and have no ordering to read.
|
||||
|
||||
Keyed to the SESSION's own vocabulary, not to HLS's: the two argv branches are disjoint, so a
|
||||
name belonging to the other shape is as hopeless here as a random string. Reading those as "an
|
||||
HLS name, so wait and see" left the whole class reachable — measured through the real route
|
||||
against a live single-audio session, `master.m3u8` and `stream_q7z.m3u8` were 404s after 30.0 s
|
||||
and `seg_a0_0.ts` / `seg_zzzz_0.ts` after 20.1 s each (deep-review round 7).
|
||||
|
||||
Saying no at once matters because the route that waits is a SYNC one: it holds a thread off the
|
||||
~40-slot pool, and a pooled DB connection with it, for the whole timeout. An ordinary seek past
|
||||
the produced region was enough to park one for 20 s, and forty concurrent requests for a name
|
||||
that will never exist parked the whole API (deep-review round 5). Round 5's guard read only
|
||||
single-audio segment names, so both halves of that were still fully reachable: every multi-audio
|
||||
name fell through, and so did any `.m3u8`/`.vtt` name at all."""
|
||||
if filename in playlist_names(s.variants):
|
||||
return True
|
||||
m = _SEG_NAME_RE.match(filename)
|
||||
if m is None:
|
||||
return False
|
||||
variant, seq = m.group(1), int(m.group(2))
|
||||
# The variant field must be one THIS session writes: present and known for a multi-audio
|
||||
# session, absent for a single-audio one.
|
||||
if variant not in (s.variants or (None,)):
|
||||
return False
|
||||
if seq == 0:
|
||||
return True
|
||||
try:
|
||||
return (s.dir / segment_name(seq - 1, variant)).exists()
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def wait_for(path: Path, timeout: float = 20.0, proc: subprocess.Popen | None = None) -> bool:
|
||||
"""Wait until a session file (playlist / segment) exists and is non-empty. Segments are
|
||||
produced ~faster than realtime, so a segment just ahead of playback appears quickly; a segment
|
||||
@@ -785,14 +942,47 @@ def sweep_orphans(older_than: float | None = None) -> int:
|
||||
return dropped
|
||||
|
||||
|
||||
def reap_idle() -> int:
|
||||
# How long a session whose ffmpeg has FINISHED may go unfetched before a new session request reaps
|
||||
# it (see reap_idle).
|
||||
_FINISHED_IDLE_S = 30
|
||||
|
||||
|
||||
def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int:
|
||||
"""Kill sessions nobody has said they still want for `_SESSION_IDLE_S` — or, once their ffmpeg
|
||||
has finished, for `finished_idle_s`.
|
||||
|
||||
A finished session is also what a PAUSED viewer has: a remux runs far faster than realtime, so
|
||||
ffmpeg is done a minute or two in and writes ENDLIST; hls.js then stops asking for the playlist,
|
||||
and with its forward buffer full a paused player fetches nothing at all. With the 30 s rule that
|
||||
killed a one-minute pause, and playback died ~30 s after resuming (deep-review round 4).
|
||||
|
||||
What makes the short rule safe again is that fetching is no longer the only signal: a player
|
||||
parked MID-ITEM checkpoints its position every 10 s, and that route calls `touch_session`, so a
|
||||
viewer who is merely paused says so far more often than this rule asks (round 5). The heartbeat
|
||||
stops once the item is finished or before a position is known (`usePlexProgress.flush`) — both
|
||||
states where reaping is correct, and where the player's 404 → reclaim recovers a viewer who
|
||||
scrubs back in anyway. Do not restate this as "every 10 s whether or not it is playing": round 6
|
||||
found that claim in two comments and not in the code. Round 4's blanket
|
||||
`reap_abandoned` bought the same safety by keeping every finished session for ten minutes — each
|
||||
one a whole segmented copy of the film on a volume shared with the download center."""
|
||||
now = time.time()
|
||||
dropped = 0
|
||||
with _lock:
|
||||
for key, s in list(_sessions.items()):
|
||||
done = s.proc.poll() is not None
|
||||
if now - s.last_access > _SESSION_IDLE_S or (done and now - s.last_access > 30):
|
||||
idle = now - s.last_access
|
||||
if idle > _SESSION_IDLE_S or (done and idle > finished_idle_s):
|
||||
_kill(s)
|
||||
_sessions.pop(key, None)
|
||||
dropped += 1
|
||||
return dropped
|
||||
|
||||
|
||||
def touch_session(key: str) -> None:
|
||||
"""Say a viewer is still there for `key`, without fetching anything. `last_access` used to move
|
||||
only on an HLS file fetch, which a paused player does not do — so the only way to keep a paused
|
||||
viewer's session alive was to keep EVERY abandoned one just as long."""
|
||||
with _lock:
|
||||
s = _sessions.get(key)
|
||||
if s is not None:
|
||||
s.last_access = time.time()
|
||||
|
||||
+730
-14
@@ -1,15 +1,57 @@
|
||||
"""WebVTT conversion, HLS-clock time-shift, and on-disk cache for Plex subtitle streams.
|
||||
"""WebVTT conversion, HLS-clock time-shift, and the on-disk cache + background extractor for Plex
|
||||
subtitle streams.
|
||||
|
||||
Text subtitles (embedded or external) are turned into WebVTT so the browser <track> can render them;
|
||||
bitmap subtitles (Blu-ray/DVD) can't be (would need burn-in). Converted VTTs are cached on the shared
|
||||
Plex on-disk cache root (``cache.CACHE_DIR``).
|
||||
|
||||
WHY THERE IS A BACKGROUND EXTRACTOR HERE — measured on prod, 2026-09-10, against a 17.06 GB 1080p
|
||||
MKV with two embedded SubRip tracks on the 81 TB array:
|
||||
|
||||
=========================================================== ========
|
||||
full pass, ONE track (what this module used to do) 154 s
|
||||
full pass, BOTH text tracks in a SINGLE ffmpeg run 111 s
|
||||
`-c:s copy` instead of converting to WebVTT 143 s
|
||||
from 1:00:00 to the end of the file 66 s
|
||||
a 630 s WINDOW at 1:00:00, both tracks 8 s
|
||||
=========================================================== ========
|
||||
|
||||
The conversion is not the cost (dropping it saved 7%): ffmpeg has to stream every byte of the
|
||||
container between its seek point and its stop point, and the array delivers ~140 MB/s — so the time
|
||||
is (bytes read) / 140 MB/s, and the ONLY lever is reading less. Hence:
|
||||
|
||||
* extract a WINDOW around the playhead first — the viewer sees subtitles in seconds — then complete
|
||||
the whole track in the background so every later seek is instant and the next session is free;
|
||||
* extract EVERY embedded text track in ONE pass, because the read is shared: picking Hungarian used
|
||||
to cost a full read and then switching to English cost another one (the "hektikus" report);
|
||||
* never do any of it inside a request (it froze the <track> with no feedback AND held an API worker
|
||||
thread for the whole read).
|
||||
|
||||
`-copyts` CANNOT be used for a window: it makes ffmpeg ignore `-t`/`-to` (measured — a [0, 600 s]
|
||||
window still took the full 145 s and emitted the whole track), so windows are extracted on a
|
||||
zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.models import PlexItem
|
||||
from app.plex import paths as plex_paths
|
||||
from app.plex.cache import CACHE_DIR
|
||||
from app.plex.client import PlexError
|
||||
from app.plex.client import PlexClient, PlexError
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
# Image-based subtitle codecs (Blu-ray/DVD) — can't be turned into text WebVTT (would need burn-in).
|
||||
BITMAP_SUBS = {
|
||||
@@ -31,6 +73,9 @@ def srt_to_vtt(text: str) -> str:
|
||||
|
||||
|
||||
def _vtt_ts_to_s(ts: str) -> float:
|
||||
# Also reads ffmpeg's spelling of a NEGATIVE time (a windowed pass, see `_ffmpeg_extract`): its
|
||||
# webvtt muxer splits the milliseconds with C's truncating division, so every field carries the
|
||||
# sign — -5.005 s is `00:-5.-05`, -65.3 s is `-1:-5.-300` — and summing the signed fields is exact.
|
||||
parts = ts.split(":")
|
||||
h = "0" if len(parts) == 2 else parts[0]
|
||||
m, rest = parts[-2], parts[-1]
|
||||
@@ -39,16 +84,12 @@ def _vtt_ts_to_s(ts: str) -> float:
|
||||
|
||||
|
||||
def _vtt_s_to_ts(x: float) -> str:
|
||||
x = max(0.0, x)
|
||||
h = int(x // 3600)
|
||||
x -= h * 3600
|
||||
m = int(x // 60)
|
||||
x -= m * 60
|
||||
s = int(x)
|
||||
ms = int(round((x - s) * 1000))
|
||||
if ms >= 1000:
|
||||
s += 1
|
||||
ms -= 1000
|
||||
# Rounded to whole milliseconds FIRST, then split: rounding the fraction last carried a rounded-up
|
||||
# millisecond into the seconds and no further, so 59.9996 came out as `00:00:60.000` — a timestamp
|
||||
# the WebVTT parser rejects, silently dropping that cue.
|
||||
h, rest = divmod(int(round(max(0.0, x) * 1000)), 3_600_000)
|
||||
m, rest = divmod(rest, 60_000)
|
||||
s, ms = divmod(rest, 1000)
|
||||
return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}"
|
||||
|
||||
|
||||
@@ -109,7 +150,7 @@ def ffmpeg_to_vtt(src: str, sub_ord: int | None) -> str:
|
||||
"webvtt",
|
||||
"pipe:1",
|
||||
]
|
||||
r = subprocess.run(cmd, capture_output=True)
|
||||
r = subprocess.run(cmd, capture_output=True, timeout=EXTRACT_TIMEOUT_S)
|
||||
if r.returncode != 0 or not r.stdout:
|
||||
raise PlexError(
|
||||
(r.stderr or b"").decode("utf-8", "replace")[:200] or "ffmpeg subtitle convert failed"
|
||||
@@ -125,11 +166,686 @@ def cache_read(key: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def cache_write(key: str, vtt: str) -> None:
|
||||
def cache_write(key: str, vtt: str) -> bool:
|
||||
"""Atomically replace the cached VTT; False when it could not be written (`publish` must then
|
||||
not claim the region it was carrying)."""
|
||||
try:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CACHE_DIR / f"{key}.vtt.tmp"
|
||||
tmp.write_text(vtt, encoding="utf-8")
|
||||
tmp.replace(CACHE_DIR / f"{key}.vtt") # atomic publish
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
# --- cache identity -----------------------------------------------------------------------------
|
||||
|
||||
# The cached VTT holds ABSOLUTE cue times (per rating_key + subtitle ordinal); the per-session shift
|
||||
# is applied on the way out, so a resume/seek to any offset reuses the same file. Beside it, a small
|
||||
# JSON sidecar records WHICH absolute-time ranges the file actually covers — a windowed extraction
|
||||
# publishes a usable but incomplete track, and "incomplete" must never be mistaken for "all there is".
|
||||
_FULL_END_S = 1_000_000_000.0 # stand-in for "to the end of the file" (JSON can't hold infinity)
|
||||
|
||||
|
||||
def cache_key(rating_key: str, sub_ord: int) -> str:
|
||||
return f"sub_{rating_key}_{sub_ord}"
|
||||
|
||||
|
||||
def _vtt_path(key: str) -> Path:
|
||||
return CACHE_DIR / f"{key}.vtt"
|
||||
|
||||
|
||||
def _meta_path(key: str) -> Path:
|
||||
return CACHE_DIR / f"{key}.json"
|
||||
|
||||
|
||||
def read_meta(key: str) -> tuple[list[list[float]], bool]:
|
||||
"""Covered absolute-time ranges + whether the track is fully extracted. A VTT with no sidecar
|
||||
(written by an older build, which only ever cached complete tracks) counts as complete."""
|
||||
try:
|
||||
raw = json.loads(_meta_path(key).read_text(encoding="utf-8"))
|
||||
return [[float(a), float(b)] for a, b in raw.get("ranges", [])], bool(raw.get("complete"))
|
||||
except (OSError, ValueError, TypeError):
|
||||
return ([[0.0, _FULL_END_S]], True) if _exists(_vtt_path(key)) else ([], False)
|
||||
|
||||
|
||||
def _exists(p: Path) -> bool:
|
||||
"""`Path.exists` that answers False for a path the OS refuses outright: pathlib swallows only
|
||||
ENOENT-like errors, so an over-long name (ENAMETOOLONG, from an absurd rating_key or ordinal)
|
||||
escaped as a 500 from the status route."""
|
||||
try:
|
||||
return p.exists()
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _write_meta(key: str, ranges: list[list[float]], complete: bool) -> None:
|
||||
try:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CACHE_DIR / f"{key}.json.tmp"
|
||||
tmp.write_text(json.dumps({"ranges": ranges, "complete": complete}), encoding="utf-8")
|
||||
tmp.replace(_meta_path(key))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def generation(key: str) -> int:
|
||||
"""Monotonic id of the cached file's current content. It goes into the <track> URL so that when a
|
||||
window is superseded by a wider one the browser refetches instead of keeping the short VTT."""
|
||||
try:
|
||||
return _vtt_path(key).stat().st_mtime_ns
|
||||
except (OSError, ValueError): # ValueError: a NUL in the key — `stat` refuses the path outright
|
||||
return 0
|
||||
|
||||
|
||||
def merge_ranges(ranges: list[list[float]]) -> list[list[float]]:
|
||||
"""Sort + coalesce touching/overlapping covered ranges."""
|
||||
out: list[list[float]] = []
|
||||
for a, b in sorted((r for r in ranges if r[1] > r[0]), key=lambda r: r[0]):
|
||||
if out and a <= out[-1][1]:
|
||||
out[-1][1] = max(out[-1][1], b)
|
||||
else:
|
||||
out.append([a, b])
|
||||
return out
|
||||
|
||||
|
||||
def covers(ranges: list[list[float]], at: float) -> bool:
|
||||
return any(a <= at <= b for a, b in ranges)
|
||||
|
||||
|
||||
def merge_vtt(*sources: str) -> str:
|
||||
"""Union of the cue blocks of several absolute-time VTTs, ordered by start time.
|
||||
|
||||
Two extraction passes over the same file emit identical cues for the region they share, so they
|
||||
are de-duplicated — but NOT on the raw timing text. The two passes do not spell a timestamp the
|
||||
same way: a window comes back through `shift_vtt`, which always writes `HH:MM:SS.mmm`, while a
|
||||
full pass is ffmpeg's own output, and its webvtt muxer drops the hour below one hour
|
||||
(`10:00.123`). Keyed on the text, every cue of a window in the first hour survived twice and the
|
||||
viewer saw each line stacked on itself. So each timing line is parsed and re-emitted in one
|
||||
canonical form, and the key is that canonical timing PLUS the cue text: two different cues that
|
||||
happen to share a time (two speakers, a sign over dialogue) are both kept."""
|
||||
seen: set[str] = set()
|
||||
cues: list[tuple[float, str]] = []
|
||||
for src in sources:
|
||||
for block in re.split(r"\n[ \t]*\n", src or ""):
|
||||
canon = _canonical_cue(block.strip())
|
||||
if canon is None:
|
||||
continue # header / NOTE / STYLE — the merged file gets its own single header
|
||||
start, key, text = canon
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
cues.append((start, text))
|
||||
cues.sort(key=lambda c: c[0])
|
||||
return "WEBVTT\n\n" + "\n\n".join(b for _, b in cues) + "\n"
|
||||
|
||||
|
||||
def _canonical_cue(block: str) -> tuple[float, str, str] | None:
|
||||
"""(start seconds, identity, the block with its timing line canonicalised) — or None for a block
|
||||
that is not a cue. The identity leaves out anything before the timing line (an SRT-style cue
|
||||
number), which is a label, not content."""
|
||||
lines = block.split("\n")
|
||||
for i, ln in enumerate(lines):
|
||||
if "-->" not in ln:
|
||||
continue
|
||||
left, _, right = ln.partition("-->")
|
||||
rparts = right.strip().split(None, 1)
|
||||
settings = (" " + rparts[1]) if len(rparts) > 1 else ""
|
||||
try:
|
||||
st = _vtt_ts_to_s(left.strip())
|
||||
en = _vtt_ts_to_s(rparts[0])
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
lines[i] = f"{_vtt_s_to_ts(st)} --> {_vtt_s_to_ts(en)}{settings}"
|
||||
return st, "\n".join(lines[i:]), "\n".join(lines)
|
||||
return None
|
||||
|
||||
|
||||
def _last_cue_end(vtt: str) -> float:
|
||||
"""Where the last cue of a VTT ends — how far a pass that died early provably got."""
|
||||
end = 0.0
|
||||
for ln in vtt.split("\n"):
|
||||
if "-->" not in ln:
|
||||
continue
|
||||
try:
|
||||
end = max(end, _vtt_ts_to_s(ln.partition("-->")[2].strip().split(None, 1)[0]))
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
return end
|
||||
|
||||
|
||||
def publish(key: str, vtt: str, covered: tuple[float, float], complete: bool) -> bool:
|
||||
"""Merge a freshly extracted region into the cached track and widen its covered ranges. False
|
||||
when the region did not land (the cache is unwritable, or the VTT there cannot be read) — the
|
||||
caller must then record a FAILED pass: returning quietly let the next poll schedule the same read
|
||||
again, a whole-container pass every time, while the player said "preparing" for good.
|
||||
|
||||
A read-merge-write, so it runs under `_PUBLISH_LOCK`: two jobs publish into the same keys (a
|
||||
window and the full pass, and every embedded track of the file rides each of them), and without
|
||||
the lock the later writer drops whatever the earlier one merged — permanently, once `complete` is
|
||||
set. It is also the only writer of the cache files, which is what keeps their fixed `.tmp`
|
||||
staging names safe."""
|
||||
with _PUBLISH_LOCK:
|
||||
old = cache_read(key)
|
||||
if old is None and _vtt_path(key).exists():
|
||||
# There, but unreadable (`cache_read` swallows the OSError). Merging into "" would replace
|
||||
# a whole track with this one region while its sidecar still claimed all of it.
|
||||
log.warning("subtitle cache %s unreadable; not publishing into it", key)
|
||||
return False
|
||||
old_ranges, was_complete = read_meta(key)
|
||||
if old is None:
|
||||
# A sidecar with no VTT behind it claims cues that are not there, so it is not merged
|
||||
# into. And a VTT with no sidecar reads as a COMPLETE track (see read_meta — how older
|
||||
# builds cached), while a publish is two renames: so before a key's first VTT lands, its
|
||||
# sidecar is made to claim nothing. Dying between the two then leaves a window that is
|
||||
# extracted again, never one taken for the whole track for good.
|
||||
old_ranges, was_complete = [], False
|
||||
_write_meta(key, [], False)
|
||||
if not cache_write(key, merge_vtt(old or "", vtt)):
|
||||
log.warning("subtitle cache %s could not be written", key)
|
||||
return False # the region never landed, so the sidecar must not claim it
|
||||
_write_meta(key, merge_ranges(old_ranges + [list(covered)]), complete or was_complete)
|
||||
return True
|
||||
|
||||
|
||||
# --- background extraction ----------------------------------------------------------------------
|
||||
|
||||
# How much of the track to extract before the viewer sees anything. WINDOW_PRE_S starts a little
|
||||
# before the playhead so a cue that is already on screen is not missed; WINDOW_LEN_S is the span
|
||||
# published as the first, provisional VTT (~4 s of work on the measured file). It only has to hold
|
||||
# until the full pass lands behind it — 111 s for that file — so it is deliberately much longer than
|
||||
# any realistic full pass.
|
||||
WINDOW_PRE_S = 30.0
|
||||
WINDOW_LEN_S = 300.0
|
||||
# …but it does not always land: a full pass can be backed off (FULL_RETRY_S) or simply slower than
|
||||
# the window lasts (a file past ~40 GB, or several passes sharing the array). A window was only asked
|
||||
# for once the playhead had LEFT coverage, so every edge unmounted the track for a poll plus an
|
||||
# extraction. Within this distance of a covered stretch's end, the next window is read ahead.
|
||||
LOOKAHEAD_S = 60.0
|
||||
# Ceiling on one ffmpeg run so a wedged extraction can't hold a pool thread forever (the old
|
||||
# in-request extraction had no timeout at all).
|
||||
EXTRACT_TIMEOUT_S = 900.0
|
||||
# How long a timed-out ffmpeg gets, after a TERM, to close its outputs before it is killed.
|
||||
_TERM_GRACE_S = 10.0
|
||||
# A failed track (or window) is remembered briefly so a 1 s status poll doesn't respawn ffmpeg every
|
||||
# second, and forgotten again soon after so fixing the file (or Plex coming back) doesn't need a
|
||||
# restart.
|
||||
FAIL_TTL_S = 60.0
|
||||
# …except a FULL pass that failed. That is a read of the whole container (minutes), a failure that
|
||||
# reproduces does so every time, and the viewer is typically inside what it did publish — so the
|
||||
# short memo re-armed a whole-file read every minute for as long as they watched. Windows around the
|
||||
# playhead still run meanwhile; only the full pass waits this out.
|
||||
FULL_RETRY_S = 3600.0
|
||||
# The furthest position the status poll accepts. No film is this long, and an absurd one used to
|
||||
# reach ffmpeg's `-ss` (which rejects anything past its int64 microseconds).
|
||||
MAX_POSITION_S = 100_000.0
|
||||
# Windows and full passes run on SEPARATE pools, so a window never waits behind a full pass. They
|
||||
# shared one FIFO pool of three whose comment promised "room for a window" that nothing enforced:
|
||||
# stepping through three episodes with subtitles on left three full passes (minutes each) holding every
|
||||
# worker, and a seek window on the third queued behind them. Two of each: bounded, so a burst can't
|
||||
# thrash the array.
|
||||
_WIN_POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="plex-subs-win")
|
||||
_FULL_POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="plex-subs-full")
|
||||
# Bounds on what may wait for those pools — separate too, so held full-pass slots (minutes each)
|
||||
# cannot refuse a window (seconds). The queues are unbounded and every job is a read of the library:
|
||||
# a viewer scrubbing through a film queued a window per stop (the one where they stopped ran LAST),
|
||||
# and a script could queue thousands. Past a bound nothing is queued — the next poll asks again, by
|
||||
# which time something has finished.
|
||||
_MAX_WINDOWS_INFLIGHT = 8
|
||||
_MAX_FULL_INFLIGHT = 6
|
||||
_MAX_TRACK_WINDOWS = 2
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
# Queued or running jobs → the absolute span a window job reads (None for a full pass).
|
||||
_INFLIGHT: dict[tuple, tuple[float, float] | None] = {}
|
||||
# Track-level failures: a track that cannot be served at all (no item, a bitmap track, Plex down…).
|
||||
_FAILURES: dict[str, tuple[str, float]] = {}
|
||||
# rating_key → when its full pass last failed (see FULL_RETRY_S).
|
||||
_FULL_FAILED: dict[str, float] = {}
|
||||
# cache key → the spans windows failed to read, with when. A failed window says nothing about the rest
|
||||
# of the track — the full pass may be about to deliver it — so it is remembered as a SPAN: it must be
|
||||
# neither respawned every poll nor taken as a failure of positions it never tried. (It used to set the
|
||||
# track-level memo, which told every viewer at any uncovered position "failed" — a final answer that
|
||||
# ends the player's poll — including one planted by a request for an absurd position.)
|
||||
_WIN_FAILED: dict[str, list[tuple[float, float, float]]] = {}
|
||||
_PUBLISH_LOCK = threading.Lock()
|
||||
# The ffmpeg children currently running, so a shutdown can kill them (see `shutdown`), and whether it
|
||||
# has: a worker that reaches its Popen after the kill sweep must not start a read.
|
||||
_PROCS: set[subprocess.Popen] = set()
|
||||
_STOPPING = False
|
||||
|
||||
|
||||
class SubtitleUnavailable(Exception):
|
||||
"""A track that cannot be served as text, carrying the stable code the route maps to a status."""
|
||||
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
# code -> HTTP status. An extraction/upstream failure is 422 with a structured detail (502 is
|
||||
# reserved for the raw byte proxies); the rest are genuine "no such thing" / "not text".
|
||||
UNAVAILABLE_STATUS = {
|
||||
"no_item": 404,
|
||||
"no_track": 404,
|
||||
"no_file": 404,
|
||||
"not_fetchable": 404,
|
||||
"bitmap": 415,
|
||||
"extract_failed": 422,
|
||||
"plex_error": 422,
|
||||
}
|
||||
# The codes that are a FINAL answer about the track: it can never be served as text, so the player
|
||||
# stops asking. The others (`plex_error`, `extract_failed`) are about now or about one stretch — a
|
||||
# memo that lapses, a window that failed where the rest is fine — and answered as `unavailable`: said,
|
||||
# but asked again. Given as `failed`, one Plex hiccup ended the subtitles for the rest of the session,
|
||||
# at cached positions too, and after the full pass had completed behind it.
|
||||
FINAL_CODES = frozenset({"no_item", "no_track", "no_file", "not_fetchable", "bitmap"})
|
||||
|
||||
|
||||
def reset_state() -> None:
|
||||
"""Drop the in-flight/failure registries (tests; nothing in the app needs it)."""
|
||||
global _STOPPING
|
||||
with _LOCK:
|
||||
_INFLIGHT.clear()
|
||||
_FAILURES.clear()
|
||||
_FULL_FAILED.clear()
|
||||
_WIN_FAILED.clear()
|
||||
_STOPPING = False
|
||||
|
||||
|
||||
def shutdown() -> None:
|
||||
"""Stop extracting, for the app's shutdown. The pool's workers are non-daemon threads and Python
|
||||
joins them at interpreter exit — so a worker still inside a full pass (minutes on a big file, up
|
||||
to EXTRACT_TIMEOUT_S) held the process open after uvicorn had stopped, until the container runtime
|
||||
SIGKILLed it. Queued jobs are dropped and running ffmpeg children are killed, which returns their
|
||||
workers at once — and a worker still BEFORE its ffmpeg (the Plex lookup, the DB) finds the flag
|
||||
and never starts one. Nothing that matters is lost: an interrupted publish cannot pass for a whole
|
||||
track (see `publish`), and an interrupted track is simply extracted again on the next poll."""
|
||||
global _STOPPING
|
||||
with _LOCK:
|
||||
_STOPPING = True
|
||||
procs = list(_PROCS)
|
||||
for pool in (_WIN_POOL, _FULL_POOL):
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
for p in procs:
|
||||
p.kill()
|
||||
|
||||
|
||||
def _sweep_memos() -> None:
|
||||
"""(Caller holds _LOCK.) Drop every lapsed failure memo. They were only ever pruned when the SAME
|
||||
key was read again, and the status route takes any rating_key/ordinal — so a script polling fresh
|
||||
keys grew the API process without bound. Swept on every write, they hold at most a TTL's worth."""
|
||||
now = time.monotonic()
|
||||
for k in [k for k, (_, t) in _FAILURES.items() if now - t >= FAIL_TTL_S]:
|
||||
del _FAILURES[k]
|
||||
for k in [k for k, t in _FULL_FAILED.items() if now - t >= FULL_RETRY_S]:
|
||||
del _FULL_FAILED[k]
|
||||
for k in list(_WIN_FAILED):
|
||||
_live_window_failures(k)
|
||||
|
||||
|
||||
def _fail(key: str, code: str) -> None:
|
||||
with _LOCK:
|
||||
_sweep_memos()
|
||||
_FAILURES[key] = (code, time.monotonic())
|
||||
|
||||
|
||||
def _recent_failure(key: str) -> str | None:
|
||||
with _LOCK:
|
||||
hit = _FAILURES.get(key)
|
||||
if hit and time.monotonic() - hit[1] < FAIL_TTL_S:
|
||||
return hit[0]
|
||||
_FAILURES.pop(key, None)
|
||||
return None
|
||||
|
||||
|
||||
def _full_failed(rating_key: str) -> bool:
|
||||
with _LOCK:
|
||||
when = _FULL_FAILED.get(rating_key)
|
||||
if when is not None and time.monotonic() - when < FULL_RETRY_S:
|
||||
return True
|
||||
_FULL_FAILED.pop(rating_key, None)
|
||||
return False
|
||||
|
||||
|
||||
def _live_window_failures(key: str) -> list[tuple[float, float, float]]:
|
||||
"""(Caller holds _LOCK.) The key's failed-window spans that are still remembered."""
|
||||
now = time.monotonic()
|
||||
live = [f for f in _WIN_FAILED.get(key, []) if now - f[2] < FAIL_TTL_S]
|
||||
if live:
|
||||
_WIN_FAILED[key] = live
|
||||
else:
|
||||
_WIN_FAILED.pop(key, None)
|
||||
return live
|
||||
|
||||
|
||||
def _window_fail(key: str, span: tuple[float, float]) -> None:
|
||||
with _LOCK:
|
||||
_sweep_memos()
|
||||
_WIN_FAILED[key] = [*_live_window_failures(key), (span[0], span[1], time.monotonic())]
|
||||
|
||||
|
||||
def _window_failed(key: str, at: float) -> bool:
|
||||
with _LOCK:
|
||||
return any(a <= at <= b for a, b, _ in _live_window_failures(key))
|
||||
|
||||
|
||||
def ensure(rating_key: str, sub_ord: int, at: float = 0.0) -> dict:
|
||||
"""State of one subtitle track, scheduling whatever extraction is still missing. NEVER blocks.
|
||||
|
||||
``ready`` = fully extracted · ``partial`` = usable, covers ``at``, still growing · ``preparing``
|
||||
= nothing to show for ``at`` yet · ``unavailable`` = nothing for ``at`` and nothing being read
|
||||
for it right now (``code`` says why), ask again · ``failed`` = the track can never be shown
|
||||
(``code`` says why) — the only final answer. ``gen`` changes whenever the cached file grows, so
|
||||
the client can refetch a window that has been superseded; 0 = nothing is cached at all."""
|
||||
key = cache_key(rating_key, sub_ord)
|
||||
ranges, complete = read_meta(key)
|
||||
have = cache_read(key) is not None
|
||||
covered = have and covers(ranges, at)
|
||||
gen = generation(key)
|
||||
if complete and have:
|
||||
return {"state": "ready", "code": None, "gen": gen}
|
||||
if not have and _exists(_vtt_path(key)):
|
||||
# On disk but unreadable (permissions, a bad volume): nothing can serve it, and no extraction
|
||||
# can help — `publish` refuses to merge into it, and a job whose sidecar covers the position
|
||||
# skips as already done — so scheduling only kept the player on "preparing" for good. `gen` 0:
|
||||
# there is nothing a <track> could fetch.
|
||||
return {"state": "unavailable", "code": "cache_unreadable", "gen": 0}
|
||||
code = _recent_failure(key)
|
||||
if code and not covered:
|
||||
return {
|
||||
"state": "failed" if code in FINAL_CODES else "unavailable",
|
||||
"code": code,
|
||||
"gen": gen,
|
||||
}
|
||||
# Nothing left that could cover the position for now: the full pass failed, AND so did a window
|
||||
# here. Still not final — the memos lapse, and the rest of the track may be right there.
|
||||
full_failed = _full_failed(rating_key)
|
||||
window_failed = not covered and _window_failed(key, at)
|
||||
if full_failed and window_failed:
|
||||
return {"state": "unavailable", "code": "extract_failed", "gen": gen}
|
||||
if not code:
|
||||
probe, want_window = at, not covered and not window_failed
|
||||
if covered:
|
||||
# Read ahead of an edge instead of waiting to fall off it (see LOOKAHEAD_S). The probe is
|
||||
# just PAST the edge: `covers` is inclusive, so the edge itself reads as already done.
|
||||
edge = max(b for a, b in ranges if a <= at <= b)
|
||||
if edge - at <= LOOKAHEAD_S and edge < _FULL_END_S:
|
||||
probe = edge + 1.0
|
||||
want_window = not _window_failed(key, probe)
|
||||
_schedule(rating_key, sub_ord, probe, want_window=want_window, want_full=not full_failed)
|
||||
return {"state": "partial" if covered else "preparing", "code": None, "gen": gen}
|
||||
|
||||
|
||||
def cached_track(rating_key: str, sub_ord: int) -> tuple[str | None, str | None]:
|
||||
"""What the <track> fetch can serve RIGHT NOW, scheduling nothing: the cached VTT (whatever it
|
||||
covers), else the recent failure that explains why there is none.
|
||||
|
||||
Scheduling belongs to `ensure` — the status poll, which knows where the viewer is. The fetch does
|
||||
not: when it scheduled too, it judged readiness at position 0, so a viewer who resumed an hour in
|
||||
got an unrequested window over the head of the file (plus a remount when it landed), and with a
|
||||
failure memo alive it was refused a window that WAS on disk while the poll called it showable."""
|
||||
key = cache_key(rating_key, sub_ord)
|
||||
hit = cache_read(key)
|
||||
return (hit, None) if hit is not None else (None, _recent_failure(key))
|
||||
|
||||
|
||||
def _schedule(
|
||||
rating_key: str, sub_ord: int, at: float, want_window: bool, want_full: bool = True
|
||||
) -> None:
|
||||
"""Kick off the jobs this track still needs: the window that makes it watchable now, and the one
|
||||
full pass that makes it complete. A 1 s status poll (or a second viewer) joins a job already
|
||||
queued or running instead of starting another ffmpeg.
|
||||
|
||||
A window is joined when one of this track's in-flight windows will COVER the position asked about
|
||||
— decided by what it reads, not by an identity derived from the position. The poll sends the live
|
||||
playhead, so a position-keyed window was a new job every second; bucketing the position instead
|
||||
still queued a second read when the playhead crossed a bucket edge, and joined a backward seek to
|
||||
a window that could not cover it. Nor may it be one window per track: a window cut around the
|
||||
wrong spot (the first poll after a resume can still report 0) made the right one wait for it. The
|
||||
full pass is per ITEM, because one read fans out to every embedded text track of the file."""
|
||||
jobs: list[tuple[tuple, float | None]] = []
|
||||
with _LOCK:
|
||||
if _STOPPING:
|
||||
return
|
||||
n_windows = sum(1 for span in _INFLIGHT.values() if span is not None)
|
||||
n_full = len(_INFLIGHT) - n_windows
|
||||
if want_window and n_windows < _MAX_WINDOWS_INFLIGHT:
|
||||
spans = [
|
||||
span
|
||||
for ident, span in _INFLIGHT.items()
|
||||
if span is not None and ident[1:3] == (rating_key, sub_ord)
|
||||
]
|
||||
if not any(a <= at <= b for a, b in spans) and len(spans) < _MAX_TRACK_WINDOWS:
|
||||
# Rounded to what ffmpeg is told (`-ss %.3f`): the cues come back on ffmpeg's zero,
|
||||
# and shifting them by an unrounded start put them up to half a millisecond off —
|
||||
# enough to spell a timestamp differently from the full pass and double the cue.
|
||||
w_start = float(f"{max(0.0, at - WINDOW_PRE_S):.3f}")
|
||||
ident = ("win", rating_key, sub_ord, w_start)
|
||||
_INFLIGHT[ident] = (w_start, w_start + WINDOW_PRE_S + WINDOW_LEN_S)
|
||||
jobs.append((ident, w_start))
|
||||
if want_full and n_full < _MAX_FULL_INFLIGHT and ("full", rating_key) not in _INFLIGHT:
|
||||
_INFLIGHT[("full", rating_key)] = None
|
||||
jobs.append((("full", rating_key), None))
|
||||
for ident, window_start in jobs:
|
||||
pool = _FULL_POOL if window_start is None else _WIN_POOL
|
||||
pool.submit(_run, ident, rating_key, sub_ord, window_start, at)
|
||||
|
||||
|
||||
def _already_done(key: str, window_start: float | None, at: float) -> bool:
|
||||
"""A queued job looks again before it reads anything. While it waited, another job may have
|
||||
covered it: every embedded track rides each pass over its file, so a window for one language also
|
||||
publishes the others."""
|
||||
if not _exists(_vtt_path(key)):
|
||||
return False
|
||||
ranges, complete = read_meta(key)
|
||||
return complete or (window_start is not None and covers(ranges, at))
|
||||
|
||||
|
||||
def _run(
|
||||
ident: tuple, rating_key: str, sub_ord: int, window_start: float | None, at: float
|
||||
) -> None:
|
||||
key = cache_key(rating_key, sub_ord)
|
||||
try:
|
||||
if _STOPPING or _already_done(key, window_start, at):
|
||||
return
|
||||
_extract_and_publish(rating_key, sub_ord, window_start)
|
||||
except SubtitleUnavailable as e:
|
||||
if e.code == "extract_failed":
|
||||
_pass_failed(rating_key, [key], window_start)
|
||||
else:
|
||||
_fail(key, e.code)
|
||||
except PlexError as e:
|
||||
log.warning("subtitle extraction failed for %s/%s: %s", rating_key, sub_ord, e)
|
||||
_fail(key, "plex_error")
|
||||
except (
|
||||
Exception
|
||||
): # a worker thread has nobody to raise to — record it, let the next poll see it
|
||||
log.exception("subtitle extraction crashed for %s/%s", rating_key, sub_ord)
|
||||
_pass_failed(rating_key, [key], window_start)
|
||||
finally:
|
||||
with _LOCK:
|
||||
_INFLIGHT.pop(ident, None)
|
||||
|
||||
|
||||
def _pass_failed(
|
||||
rating_key: str,
|
||||
keys: list[str],
|
||||
window_start: float | None,
|
||||
span: tuple[float, float] | None = None,
|
||||
) -> None:
|
||||
"""Record an ffmpeg pass that failed: a full pass backs off the whole item (FULL_RETRY_S); a
|
||||
window marks only the stretch it failed to read, for the tracks it was reading."""
|
||||
if window_start is None:
|
||||
with _LOCK:
|
||||
_sweep_memos()
|
||||
_FULL_FAILED[rating_key] = time.monotonic()
|
||||
return
|
||||
for key in keys:
|
||||
_window_fail(key, span or (window_start, window_start + WINDOW_PRE_S + WINDOW_LEN_S))
|
||||
|
||||
|
||||
def _text_streams(db: Session, rating_key: str) -> tuple[PlexItem, list[dict]]:
|
||||
"""The item plus its subtitle streams, in the ordinal order the API and the UI both use."""
|
||||
it = by_rating_key(db, PlexItem, rating_key)
|
||||
if it is None:
|
||||
raise SubtitleUnavailable("no_item")
|
||||
with PlexClient(db) as plex:
|
||||
part = ((plex.metadata(it.rating_key) or {}).get("Media") or [{}])[0].get("Part") or [{}]
|
||||
return it, [s for s in (part[0].get("Stream") or []) if s.get("streamType") == 3]
|
||||
|
||||
|
||||
def _is_bitmap(stream: dict) -> bool:
|
||||
return (stream.get("codec") or stream.get("format") or "").lower() in BITMAP_SUBS
|
||||
|
||||
|
||||
def _extract_and_publish(rating_key: str, sub_ord: int, window_start: float | None) -> None:
|
||||
"""One extraction job. A sidecar track is fetched whole from Plex (cheap, no ffmpeg); an embedded
|
||||
one is extracted together with EVERY other embedded text track in the file, because they all ride
|
||||
on the same single read — the whole point of the measurement at the top of this module."""
|
||||
with SessionLocal() as db:
|
||||
it, subs = _text_streams(db, rating_key)
|
||||
if sub_ord < 0 or sub_ord >= len(subs):
|
||||
raise SubtitleUnavailable("no_track")
|
||||
want = subs[sub_ord]
|
||||
if _is_bitmap(want):
|
||||
raise SubtitleUnavailable("bitmap")
|
||||
if want.get("key"):
|
||||
# An external sidecar is one small fetch from Plex, never a read of the media file, so a
|
||||
# window job fetches the whole thing too. It must not be left to the full job: that
|
||||
# identity is per ITEM (the embedded fan-out shares it), so a sidecar picked while an
|
||||
# embedded track's full pass was running sat behind that pass for minutes.
|
||||
vtt = _sidecar_vtt(db, want)
|
||||
if not publish(cache_key(rating_key, sub_ord), vtt, (0.0, _FULL_END_S), True):
|
||||
raise SubtitleUnavailable("extract_failed")
|
||||
return
|
||||
if want.get("index") is None:
|
||||
raise SubtitleUnavailable("not_fetchable")
|
||||
src = plex_paths.local_media_path(db, it.file_path)
|
||||
if src is None:
|
||||
raise SubtitleUnavailable("no_file")
|
||||
# Ordinal among the file's EMBEDDED subtitle streams — what `-map 0:s:N` counts. Sidecars sit
|
||||
# in the same list but not in the file, so they must not shift the mapping.
|
||||
targets = {
|
||||
sum(1 for x in subs[:i] if x.get("index") is not None): i
|
||||
for i, s in enumerate(subs)
|
||||
if s.get("index") is not None and not _is_bitmap(s)
|
||||
}
|
||||
start = window_start or 0.0
|
||||
length = (WINDOW_LEN_S + WINDOW_PRE_S) if window_start is not None else None
|
||||
end = _FULL_END_S if length is None else start + length
|
||||
texts, clean = _ffmpeg_extract(str(src), sorted(targets), window_start, length)
|
||||
for emb_ord, vtt in texts.items():
|
||||
key = cache_key(rating_key, targets[emb_ord])
|
||||
# A pass that exited non-zero holds only what ffmpeg reached before it died. Claim no more
|
||||
# than its last cue, never call it complete, and remember the failure: published as complete,
|
||||
# the track read "ready" for good while its subtitles stopped wherever ffmpeg did (the cache
|
||||
# has no eviction, so nothing would ever have extracted it again).
|
||||
covered_end = end if clean else start + _last_cue_end(vtt)
|
||||
landed = publish(
|
||||
key,
|
||||
shift_vtt(vtt, -start), # zero-based window clock → absolute cue times
|
||||
(start, covered_end),
|
||||
clean and length is None,
|
||||
)
|
||||
if not landed:
|
||||
# A read that could not be kept is a failed read (see publish) — backed off like one.
|
||||
_pass_failed(rating_key, [key], window_start, (start, end))
|
||||
elif not clean:
|
||||
_pass_failed(rating_key, [key], window_start, (covered_end, end))
|
||||
|
||||
|
||||
def _sidecar_vtt(db: Session, stream: dict) -> str:
|
||||
"""An external subtitle file, fetched through Plex and converted to WebVTT."""
|
||||
with PlexClient(db) as plex:
|
||||
raw = plex.raw_get(stream["key"])
|
||||
codec = (stream.get("codec") or stream.get("format") or "").lower()
|
||||
if codec in ("srt", "subrip", ""):
|
||||
return srt_to_vtt(raw.decode("utf-8", "replace"))
|
||||
with tempfile.NamedTemporaryFile(suffix="." + (codec or "sub"), delete=False) as fh:
|
||||
fh.write(raw)
|
||||
tmp = fh.name
|
||||
try:
|
||||
return ffmpeg_to_vtt(tmp, None)
|
||||
finally:
|
||||
Path(tmp).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _ffmpeg_extract(
|
||||
src: str, emb_ords: list[int], start: float | None, length: float | None
|
||||
) -> tuple[dict[int, str], bool]:
|
||||
"""Extract several embedded subtitle streams to WebVTT in ONE ffmpeg run (= one read of the file).
|
||||
Returns the texts and whether ffmpeg exited cleanly — a non-zero exit can still leave partial
|
||||
output, which is worth publishing but must never be taken for the whole track.
|
||||
|
||||
`start`/`length` bound the read to a window. Without `-copyts` the cues come out on a zero-based
|
||||
clock (the caller shifts them back) — which is exactly why `-copyts` is absent: with it ffmpeg
|
||||
ignores `-t` and reads the whole container anyway (measured)."""
|
||||
if not emb_ords:
|
||||
return {}, True
|
||||
with tempfile.TemporaryDirectory(prefix="plexsub-") as tmpdir:
|
||||
cmd = ["ffmpeg", "-nostdin", "-loglevel", "error"]
|
||||
if start:
|
||||
cmd += ["-ss", f"{start:.3f}"]
|
||||
cmd += ["-i", src]
|
||||
outs = {o: Path(tmpdir) / f"{o}.vtt" for o in emb_ords}
|
||||
for emb_ord, out in outs.items():
|
||||
cmd += ["-map", f"0:s:{emb_ord}"]
|
||||
if length is not None:
|
||||
cmd += ["-t", f"{length:.3f}"]
|
||||
# The input seek lands on the keyframe at or BEFORE `start`, so a cue already running
|
||||
# at `start` comes out at a negative time — and by default the muxer then shifts the
|
||||
# WHOLE output so that cue sits at 0: every cue of the window late by the distance to
|
||||
# that keyframe (1–5 s measured on a real library file, a whole GOP on a synthetic
|
||||
# one), and doubled for good once the full pass merged in beside it. Kept negative, the
|
||||
# times are exact on the zero-based clock (`_vtt_ts_to_s` reads the sign).
|
||||
cmd += ["-avoid_negative_ts", "disabled"]
|
||||
cmd += ["-f", "webvtt", "-y", str(out)]
|
||||
# Popen rather than run(): the child is registered so `shutdown` can kill it — and one that
|
||||
# would start after the shutdown's kill sweep never does, or is killed right here.
|
||||
with _LOCK:
|
||||
if _STOPPING:
|
||||
raise SubtitleUnavailable("extract_failed")
|
||||
proc = subprocess.Popen(
|
||||
cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
|
||||
)
|
||||
with _LOCK:
|
||||
_PROCS.add(proc)
|
||||
if _STOPPING:
|
||||
proc.kill()
|
||||
timed_out = False
|
||||
try:
|
||||
try:
|
||||
_, err = proc.communicate(timeout=EXTRACT_TIMEOUT_S)
|
||||
except subprocess.TimeoutExpired:
|
||||
# A pass that ran out of time is a pass that died partway: what it reached is kept and
|
||||
# published as partial (the caller's unclean path), like any other non-zero exit. It
|
||||
# used to be killed and its outputs deleted unread — and with the full-pass backoff, the
|
||||
# same doomed read came back every hour. TERM first: ffmpeg closes its outputs on it,
|
||||
# whereas a KILL leaves only what was already flushed in whole blocks (measured: none).
|
||||
timed_out = True
|
||||
proc.terminate()
|
||||
try:
|
||||
_, err = proc.communicate(timeout=_TERM_GRACE_S)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
_, err = proc.communicate()
|
||||
finally:
|
||||
with _LOCK:
|
||||
_PROCS.discard(proc)
|
||||
got = {
|
||||
o: p.read_text(encoding="utf-8", errors="replace")
|
||||
for o, p in outs.items()
|
||||
if p.exists()
|
||||
}
|
||||
if not got:
|
||||
raise SubtitleUnavailable("extract_failed")
|
||||
clean = proc.returncode == 0 and not timed_out
|
||||
if not clean:
|
||||
log.warning(
|
||||
"ffmpeg subtitle pass %s: %s",
|
||||
"timed out" if timed_out else f"exited {proc.returncode}",
|
||||
(err or b"")[:200],
|
||||
)
|
||||
return got, clean
|
||||
|
||||
+65
-11
@@ -1,21 +1,30 @@
|
||||
"""A tiny in-process sliding-window rate limiter.
|
||||
|
||||
Generic groundwork: keyed by an arbitrary string (e.g. a client IP), so it can throttle any
|
||||
Generic groundwork: keyed by an arbitrary string (e.g. a client — `client_key`), so it can throttle any
|
||||
endpoint. Single-worker uvicorn → in-memory state is sufficient; it resets on restart, which
|
||||
is fine for abuse throttling (not for anything that must survive a deploy). Not shared across
|
||||
processes — if we ever run multiple workers, swap the backing store for Redis behind the same
|
||||
``allow()`` interface.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import threading
|
||||
import time
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
def __init__(self, max_events: int, window_seconds: float):
|
||||
"""At most `max_events` attempts per `window_seconds` per key — or, when each attempt passes a
|
||||
`cost`, at most that much total cost (a byte budget counts bytes, not calls)."""
|
||||
|
||||
def __init__(self, max_events: float, window_seconds: float):
|
||||
self.max_events = max_events
|
||||
self.window = window_seconds
|
||||
self._hits: dict[str, list[float]] = {}
|
||||
# key → (monotonic time, cost) of every counted attempt still inside the window
|
||||
self._hits: dict[str, list[tuple[float, float]]] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def reset(self) -> None:
|
||||
@@ -25,27 +34,72 @@ class RateLimiter:
|
||||
with self._lock:
|
||||
self._hits.clear()
|
||||
|
||||
def allow(self, key: str) -> bool:
|
||||
"""Record an attempt for ``key`` and return whether it is within the limit.
|
||||
def exhausted(self, key: str) -> bool:
|
||||
"""Whether ``key``'s window is already full (``allow`` would refuse it), WITHOUT counting an
|
||||
attempt — for a limit that counts only some outcomes: ask it first, ``allow`` the ones that
|
||||
count."""
|
||||
cutoff = time.monotonic() - self.window
|
||||
with self._lock:
|
||||
return sum(c for t, c in self._hits.get(key, []) if t > cutoff) >= self.max_events
|
||||
|
||||
True -> under the cap (the attempt is counted).
|
||||
False -> the cap for the current window is already reached (attempt NOT counted, so a
|
||||
def allow(self, key: str, cost: float = 1) -> bool:
|
||||
"""Record an attempt costing ``cost`` for ``key`` and return whether it is within the limit.
|
||||
|
||||
True -> within the cap (the attempt is counted).
|
||||
False -> the attempt would pass the cap for the current window (attempt NOT counted, so a
|
||||
blocked caller can't keep pushing the window forward)."""
|
||||
now = time.monotonic()
|
||||
cutoff = now - self.window
|
||||
with self._lock:
|
||||
hits = [t for t in self._hits.get(key, []) if t > cutoff]
|
||||
if len(hits) >= self.max_events:
|
||||
hits = [(t, c) for t, c in self._hits.get(key, []) if t > cutoff]
|
||||
if sum(c for _, c in hits) + cost > self.max_events:
|
||||
self._hits[key] = hits
|
||||
return False
|
||||
hits.append(now)
|
||||
hits.append((now, cost))
|
||||
self._hits[key] = hits
|
||||
# Opportunistic cleanup so abandoned keys don't accumulate unboundedly.
|
||||
if len(self._hits) > 4096:
|
||||
for k in list(self._hits):
|
||||
fresh = [t for t in self._hits[k] if t > cutoff]
|
||||
fresh = [(t, c) for t, c in self._hits[k] if t > cutoff]
|
||||
if fresh:
|
||||
self._hits[k] = fresh
|
||||
else:
|
||||
del self._hits[k]
|
||||
return True
|
||||
|
||||
|
||||
def _norm_ip(s: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
||||
"""Parse an IP for comparison, unwrapping IPv4-mapped IPv6 (`::ffff:10.0.0.1` → `10.0.0.1`) so a
|
||||
plain-IPv4 allowlist still matches a peer that surfaces in mapped form. None for a non-IP string."""
|
||||
try:
|
||||
ip = ipaddress.ip_address(s.strip())
|
||||
except ValueError:
|
||||
return None
|
||||
return ip.ipv4_mapped if getattr(ip, "ipv4_mapped", None) else ip
|
||||
|
||||
|
||||
def client_key(request: Request) -> str:
|
||||
"""Best-effort client identity for rate limiting: its IP, or for IPv6 the /64 the address sits in — a
|
||||
subscriber is handed a whole /64, so keyed on the full address one client would have endless identities
|
||||
to spread its attempts over. X-Forwarded-For is client-controlled, so we trust it
|
||||
ONLY when the request actually arrived from a configured reverse proxy (settings.trusted_proxy_ips
|
||||
— the address the proxy connects FROM, e.g. the VPS Caddy's WireGuard peer IP). Our proxy APPENDS
|
||||
the client it saw to XFF, so the RIGHTMOST entry is the real client and can't be forged by a
|
||||
client pre-seeding the header. Any other peer (someone hitting the app port directly, bypassing the
|
||||
proxy) is untrusted — we use its real socket IP, so it can't spoof its rate-limit identity.
|
||||
|
||||
SAFETY: this relies on `request.client` being the TRUE socket peer. uvicorn is started WITHOUT
|
||||
`--proxy-headers` (see entrypoint.sh) precisely so it never rewrites `request.client` from XFF —
|
||||
do NOT add that flag, or the trust check below would compare against a forgeable value."""
|
||||
peer = request.client.host if request.client else "unknown"
|
||||
client = peer
|
||||
trusted = {_norm_ip(p) for p in settings.trusted_proxy_ip_set} - {None}
|
||||
if _norm_ip(peer) in trusted:
|
||||
xff = request.headers.get("x-forwarded-for")
|
||||
parts = [p.strip() for p in (xff or "").split(",") if p.strip()]
|
||||
if parts:
|
||||
client = parts[-1]
|
||||
ip = _norm_ip(client)
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
return str(ipaddress.IPv6Network((ip, 64), strict=False))
|
||||
return client
|
||||
|
||||
@@ -17,12 +17,20 @@ from app.models import User
|
||||
M = TypeVar("M", bound=Base)
|
||||
|
||||
|
||||
def get_or_404(db: Session, model: type[M], id_: Any, *, detail: str = "Not found") -> M:
|
||||
def not_found(what: str = "generic") -> HTTPException:
|
||||
"""The 404 for a row that is gone — structured, so the bilingual UI says it in the reader's
|
||||
language (`errors.notFound.<what>`, generic fallback). This is the 404 a user actually meets: the
|
||||
thing was deleted in another tab or on another device. It used to be a bare English string
|
||||
("Unknown view", "Not found"), which the Hungarian UI toasted verbatim."""
|
||||
return HTTPException(status_code=404, detail={"code": "not_found", "reason": what})
|
||||
|
||||
|
||||
def get_or_404(db: Session, model: type[M], id_: Any, *, what: str = "generic") -> M:
|
||||
"""Fetch a row by primary key or raise 404. Replaces the `db.get(Model, id); if None: 404`
|
||||
idiom repeated across routes."""
|
||||
obj = db.get(model, id_)
|
||||
if obj is None:
|
||||
raise HTTPException(status_code=404, detail=detail)
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
|
||||
@@ -33,13 +41,13 @@ def owned_or_404(
|
||||
user: User,
|
||||
*,
|
||||
owner_attr: str = "user_id",
|
||||
detail: str = "Not found",
|
||||
what: str = "generic",
|
||||
) -> M:
|
||||
"""Fetch a row by primary key that `user` owns, or raise 404. A row owned by someone
|
||||
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
||||
Generalizes the near-identical per-route ownership helpers (e.g. plex's
|
||||
`_own_playlist_or_404`, the saved-views owner check)."""
|
||||
obj = get_or_404(db, model, id_, detail=detail)
|
||||
obj = get_or_404(db, model, id_, what=what)
|
||||
if getattr(obj, owner_attr) != user.id:
|
||||
raise HTTPException(status_code=404, detail=detail)
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
+12
-16
@@ -173,7 +173,7 @@ def _activate_user_by_email(db: Session, email: str) -> None:
|
||||
|
||||
|
||||
def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
inv = get_or_404(db, Invite, invite_id, detail="No such invite")
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
# Idempotency guard: only a still-pending invite can be decided. Without this a repeat/stale
|
||||
# approve or deny re-runs its side effects — re-activating a user, or (now that deny emails)
|
||||
# sending a duplicate decision mail, even flipping an already-approved invite to "denied" and
|
||||
@@ -291,7 +291,7 @@ def send_invitation(
|
||||
|
||||
Only an APPROVED row can be invited: there is nothing to invite someone to while their request is
|
||||
undecided, and inviting a denied address would contradict the decision."""
|
||||
inv = get_or_404(db, Invite, invite_id, detail="No such invite")
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
if inv.status != "approved":
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -364,7 +364,7 @@ def delete_invite(
|
||||
falls back to the env ALLOWED_EMAILS / ADMIN_EMAILS bootstrap lists once no invite exists, so for
|
||||
an address on those the deletion leaves access intact (and an ADMIN_EMAILS address is still
|
||||
promoted to admin on its next Google sign-in). The admin has to hear that, not guess it."""
|
||||
inv = get_or_404(db, Invite, invite_id, detail="No such invite")
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
if _accounts_for(db, [inv.email]):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -408,6 +408,8 @@ def _serialize_user(u: User) -> dict:
|
||||
# confusing sign-in.
|
||||
"google_email": u.google_email if u.google_email != u.email else None,
|
||||
"created_at": iso(u.created_at),
|
||||
# NULL = no sign-in since the column existed (0085), or the shared demo account.
|
||||
"last_login_at": iso(u.last_login_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -434,9 +436,7 @@ def set_user_role(
|
||||
role = body.role
|
||||
if role not in ("user", "admin"):
|
||||
raise HTTPException(status_code=400, detail="role must be 'user' or 'admin'")
|
||||
target = db.get(User, user_id)
|
||||
if target is None:
|
||||
raise HTTPException(status_code=404, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account's role can't be changed.")
|
||||
if target.id == admin.id:
|
||||
@@ -491,9 +491,7 @@ def set_user_suspended(
|
||||
un-suspend the user is emailed they can return (suspend is announced reactively, on a blocked
|
||||
sign-in)."""
|
||||
suspended = body.suspended
|
||||
target = db.get(User, user_id)
|
||||
if target is None:
|
||||
raise HTTPException(status_code=404, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be suspended.")
|
||||
if target.id == admin.id:
|
||||
@@ -537,9 +535,7 @@ def admin_delete_user(
|
||||
"""Admin-triggered account deletion — the same full erasure a user can perform on themselves
|
||||
(cascade delete of all personal data + access-request cleanup + Google-grant revoke). Guards:
|
||||
not the demo account, not yourself (use Settings → Account), and not the last admin."""
|
||||
target = db.get(User, user_id)
|
||||
if target is None:
|
||||
raise HTTPException(status_code=404, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be deleted.")
|
||||
if target.id == admin.id:
|
||||
@@ -632,7 +628,7 @@ def user_access(
|
||||
Its own endpoint rather than extra columns on /users: `is_allowed` and `can_activate` each run a
|
||||
query, and the panel opens for ONE user at a time — folding them into the list would pay that cost
|
||||
per row on every load to answer a question nobody asked yet."""
|
||||
target = get_or_404(db, User, user_id, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
return _serialize_user(target) | {
|
||||
"is_allowed": is_allowed(db, target.email),
|
||||
"can_activate": can_activate(db, target),
|
||||
@@ -657,7 +653,7 @@ def create_set_password_link(
|
||||
an instance with no mail — narrowing it to one of those would only make the admin's real case the
|
||||
one it refuses. The demo account is the single exception: it is shared, so handing anyone a way to
|
||||
change its password is handing them everyone else's session."""
|
||||
target = get_or_404(db, User, user_id, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "setpw", "reason": "demo", "email": target.email}
|
||||
@@ -693,7 +689,7 @@ def activate_user(
|
||||
This endpoint changes nothing about what an unauthenticated caller can cause — the admin could
|
||||
already do exactly this by re-entering the address in the whitelist box (`add_invite` →
|
||||
`_activate_user_by_email`). What was missing was any way to know they had to."""
|
||||
target = get_or_404(db, User, user_id, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be activated.")
|
||||
if target.is_active:
|
||||
@@ -732,7 +728,7 @@ def resend_user_verification(
|
||||
|
||||
Not rate-limited: an admin is not a threat model to enumerate against, and the deliberate silence
|
||||
of the public endpoint is exactly what this exists to escape."""
|
||||
target = get_or_404(db, User, user_id, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo or target.email_verified:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
|
||||
+21
-53
@@ -3,7 +3,6 @@
|
||||
Split out of app/auth.py (R12 S2); the core auth surface (current_user, scopes, token
|
||||
helpers) stays in app/auth.py, imported below."""
|
||||
|
||||
import ipaddress
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import NamedTuple
|
||||
@@ -40,7 +39,7 @@ from app.auth import (
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import AuthToken, DemoWhitelist, Invite, OAuthToken, User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import encrypt, hash_password, hash_token, verify_password
|
||||
from app.utils import canonical_email, valid_email
|
||||
|
||||
@@ -172,37 +171,6 @@ def is_demo_allowed(db: Session, email: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _norm_ip(s: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
||||
"""Parse an IP for comparison, unwrapping IPv4-mapped IPv6 (`::ffff:10.0.0.1` → `10.0.0.1`) so a
|
||||
plain-IPv4 allowlist still matches a peer that surfaces in mapped form. None for a non-IP string."""
|
||||
try:
|
||||
ip = ipaddress.ip_address(s.strip())
|
||||
except ValueError:
|
||||
return None
|
||||
return ip.ipv4_mapped if getattr(ip, "ipv4_mapped", None) else ip
|
||||
|
||||
|
||||
def _client_ip(request: Request) -> str:
|
||||
"""Best-effort client IP for rate limiting. X-Forwarded-For is client-controlled, so we trust it
|
||||
ONLY when the request actually arrived from a configured reverse proxy (settings.trusted_proxy_ips
|
||||
— the address the proxy connects FROM, e.g. the VPS Caddy's WireGuard peer IP). Our proxy APPENDS
|
||||
the client it saw to XFF, so the RIGHTMOST entry is the real client and can't be forged by a
|
||||
client pre-seeding the header. Any other peer (someone hitting the app port directly, bypassing the
|
||||
proxy) is untrusted — we use its real socket IP, so it can't spoof its rate-limit identity.
|
||||
|
||||
SAFETY: this relies on `request.client` being the TRUE socket peer. uvicorn is started WITHOUT
|
||||
`--proxy-headers` (see entrypoint.sh) precisely so it never rewrites `request.client` from XFF —
|
||||
do NOT add that flag, or the trust check below would compare against a forgeable value."""
|
||||
peer = request.client.host if request.client else "unknown"
|
||||
trusted = {_norm_ip(p) for p in settings.trusted_proxy_ip_set} - {None}
|
||||
if _norm_ip(peer) in trusted:
|
||||
xff = request.headers.get("x-forwarded-for")
|
||||
parts = [p.strip() for p in (xff or "").split(",") if p.strip()]
|
||||
if parts:
|
||||
return parts[-1]
|
||||
return peer
|
||||
|
||||
|
||||
class InviteRequest(NamedTuple):
|
||||
"""What an access request settled on.
|
||||
|
||||
@@ -511,16 +479,8 @@ async def callback(
|
||||
db.flush()
|
||||
|
||||
_store_token(db, user, token)
|
||||
_establish_session(request, user)
|
||||
db.commit()
|
||||
|
||||
# Multi-session: remember every account that has authenticated in this browser so the
|
||||
# user can switch between them without going through Google again. Only accounts that
|
||||
# actually completed OAuth here land in this list (it's the proof they may be switched to).
|
||||
accounts = [a for a in (request.session.get("account_ids") or []) if a != user.id]
|
||||
accounts.append(user.id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
request.session["user_id"] = user.id
|
||||
_remember_epoch(request, user)
|
||||
log.info("Login: %s (id=%s, role=%s)", email, user.id, user.role)
|
||||
return RedirectResponse(url="/")
|
||||
|
||||
@@ -631,6 +591,8 @@ async def logout(request: Request):
|
||||
out in one tab doesn't disturb another tab running a different account. If the account being
|
||||
removed was the session default, promote the most recent remaining account (or clear the
|
||||
session when none remain)."""
|
||||
# A tab still pinned to an account already signed out in another tab resolves to nobody, so it
|
||||
# signs nobody out — it used to fall back to the default and sign THAT account out in its place.
|
||||
active, is_default = resolved_user_id(request)
|
||||
remaining = [a for a in (request.session.get("account_ids") or []) if a != active]
|
||||
request.session["account_ids"] = remaining
|
||||
@@ -672,7 +634,7 @@ async def request_access(
|
||||
email = (body.email or "").strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
if not _request_access_limiter.allow(_client_ip(request)):
|
||||
if not _request_access_limiter.allow(client_key(request)):
|
||||
return {"status": "pending"}
|
||||
if is_allowed(db, email):
|
||||
return {"status": "pending"} # already allowed — don't reveal it
|
||||
@@ -689,7 +651,7 @@ def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -
|
||||
so there is no visible 'demo login' button. Rate-limited per IP; when blocked it answers
|
||||
exactly like a non-match (authenticated=false) so the field can't be hammered as an
|
||||
enumeration oracle. On a match it sets the session and the client reloads into the app."""
|
||||
if not _demo_limiter.allow(_client_ip(request)):
|
||||
if not _demo_limiter.allow(client_key(request)):
|
||||
return {"authenticated": False}
|
||||
email = (body.email or "").strip().lower()
|
||||
if valid_email(email) and is_demo_allowed(db, email):
|
||||
@@ -705,13 +667,18 @@ def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -
|
||||
|
||||
|
||||
def _establish_session(request: Request, user: User) -> None:
|
||||
"""Sign `user` in for this browser session, mirroring the Google callback's multi-account
|
||||
handling so password sign-in joins the same account switcher."""
|
||||
"""Sign `user` in for this browser session — the ONE sign-in path, shared by the Google callback
|
||||
and password login, so both join the same account switcher and both stamp `last_login_at` (the
|
||||
caller commits). Multi-session: every account that authenticated in this browser is remembered so
|
||||
the user can switch between them; only accounts that actually proved themselves HERE land in that
|
||||
wallet (it is the proof they may be switched to). The demo entry deliberately does not come
|
||||
through here: it clears the wallet instead, and its shared account has no meaningful last login."""
|
||||
accounts = [a for a in (request.session.get("account_ids") or []) if a != user.id]
|
||||
accounts.append(user.id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
request.session["user_id"] = user.id
|
||||
_remember_epoch(request, user)
|
||||
user.last_login_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _app_base() -> str:
|
||||
@@ -835,7 +802,7 @@ def register(
|
||||
)
|
||||
if not sysconfig.get_bool(db, "allow_registration"):
|
||||
raise HTTPException(status_code=403, detail="Registration is currently closed.")
|
||||
if not _register_limiter.allow(_client_ip(request)):
|
||||
if not _register_limiter.allow(client_key(request)):
|
||||
return {"status": "ok"} # silently throttle; uniform response
|
||||
|
||||
# Do the existence check + account creation OFF the response path (a background task with its own
|
||||
@@ -1003,7 +970,7 @@ def verify_email_resend(
|
||||
and re-registering an already-registered email is a silent no-op, so a user who lost the first
|
||||
mail had no way back. Uniform response + timing regardless of whether the email has a pending
|
||||
account (the lookup runs off the response path), so it can't probe for registered emails."""
|
||||
if not _verify_resend_limiter.allow(_client_ip(request)):
|
||||
if not _verify_resend_limiter.allow(client_key(request)):
|
||||
return {"status": "ok"} # silently throttle; uniform response
|
||||
email = (body.email or "").strip().lower()
|
||||
if valid_email(email):
|
||||
@@ -1062,7 +1029,7 @@ def password_login(
|
||||
"""Email+password sign-in. Wrong email/password give a single uniform 401 (no enumeration).
|
||||
Once the password is proven correct, the owner gets a specific reason if their account is
|
||||
still pending or suspended — that's not an enumeration leak (they already hold the password)."""
|
||||
if not _login_limiter.allow(_client_ip(request)):
|
||||
if not _login_limiter.allow(client_key(request)):
|
||||
raise _login_refused(429, "rate_limited")
|
||||
email = (body.email or "").strip().lower()
|
||||
password = body.password or ""
|
||||
@@ -1081,6 +1048,7 @@ def password_login(
|
||||
if not user.is_active:
|
||||
raise _login_refused(403, "pending_approval", user)
|
||||
_establish_session(request, user)
|
||||
db.commit()
|
||||
log.info("Password login: %s (id=%s, role=%s)", email, user.id, user.role)
|
||||
return {"ok": True}
|
||||
|
||||
@@ -1093,7 +1061,7 @@ def password_reset_request(
|
||||
) -> dict:
|
||||
"""Request a password-reset link. Uniform response + timing regardless of whether the email has a
|
||||
password account (the lookup runs off the response path), so it can't probe for registered emails."""
|
||||
if not _reset_limiter.allow(_client_ip(request)):
|
||||
if not _reset_limiter.allow(client_key(request)):
|
||||
return {"status": "ok"}
|
||||
email = (body.email or "").strip().lower()
|
||||
# Do the account lookup + token issue + email OFF the response path (a background task with its
|
||||
@@ -1142,7 +1110,7 @@ def password_reset_check(body: TokenIn, request: Request, db: Session = Depends(
|
||||
told `true`, never `false`: this check may only ever save a user a wasted form, so when it cannot
|
||||
answer it must degrade to the old behaviour instead of declaring a live link dead.
|
||||
"""
|
||||
if not _reset_check_limiter.allow(_client_ip(request)):
|
||||
if not _reset_check_limiter.allow(client_key(request)):
|
||||
return {"ok": True}
|
||||
return {"ok": any(_peek_token(db, body.token, k) is not None for k in _PASSWORD_TOKEN_KINDS)}
|
||||
|
||||
@@ -1246,7 +1214,7 @@ def logout_others(
|
||||
are rejected, then re-stamp THIS cookie so the caller stays signed in. For "I left myself logged
|
||||
in somewhere" / a suspected session compromise. The demo account (shared) is excluded."""
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="Not available in the demo account.")
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
# Commit the bump BEFORE re-stamping this cookie, so a failed commit can't strand the current
|
||||
# session at a newer epoch than the DB (see set_password).
|
||||
bump_session_epoch(user)
|
||||
@@ -1293,7 +1261,7 @@ def set_password(
|
||||
account) needs no current password — the session already proves identity. Changing an
|
||||
existing one requires the current password, so a hijacked session can't silently rotate it."""
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="Not available in the demo account.")
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
new_password = body.password or ""
|
||||
if len(new_password) < PASSWORD_MIN_LEN:
|
||||
raise HTTPException(
|
||||
|
||||
@@ -26,7 +26,7 @@ from app.models import (
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.routes._common import owned_or_404
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404
|
||||
from app.sync.explore import explore_ingest_page
|
||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
@@ -287,9 +287,7 @@ def channel_detail(
|
||||
needed — a stub, or a channel last enriched before those fields existed (published_at is a
|
||||
reliable sentinel: every real channel has a creation date). The enrich uses the API key
|
||||
(1 unit, cached after), so it's skipped for the quota-less demo account."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if channel.published_at is None and not user.is_demo:
|
||||
try:
|
||||
with (
|
||||
@@ -366,9 +364,7 @@ def explore_channel(
|
||||
it, then ingests one page of its uploads (newest-first) marked via_explore + enriched.
|
||||
Returns the next page token so the page can "load more". Guarded against the shared
|
||||
backfill quota reserve; blocked for the demo account (it spends shared quota)."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if _is_blocked(db, user, channel_id):
|
||||
raise HTTPException(status_code=403, detail="You've blocked this channel.")
|
||||
if quota.backfill_reserve_reached():
|
||||
@@ -426,7 +422,7 @@ def _user_subscription(db: Session, user: User, channel_id: str) -> Subscription
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if sub is None:
|
||||
raise HTTPException(status_code=404, detail="Not subscribed to this channel")
|
||||
raise not_found("subscription")
|
||||
return sub
|
||||
|
||||
|
||||
@@ -512,9 +508,7 @@ def reset_backfill(
|
||||
state (a "reset" trigger). Clears the channel's backfill markers, opts it back into deep
|
||||
backfill, and re-runs its recent pull immediately; the deep scheduler re-pages the full
|
||||
back-catalog on its next run. Idempotent — videos upsert by id, so nothing duplicates."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
channel.backfill_done = False
|
||||
channel.backfill_cursor = None
|
||||
channel.recent_synced_at = None
|
||||
@@ -544,9 +538,7 @@ def subscribe(
|
||||
local subscription with YouTube's returned resource id and enriches the channel's metadata
|
||||
if it's still a stub, so the new row renders properly in the manager. It does NOT pull the
|
||||
channel's videos — the scheduler picks those up on its next run, like any subscribed channel."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
existing = db.execute(
|
||||
select(Subscription).where(
|
||||
Subscription.user_id == user.id, Subscription.channel_id == channel_id
|
||||
@@ -663,8 +655,7 @@ def block_channel(
|
||||
"""Block a channel for this user: its videos won't appear in their live search (and won't be
|
||||
ingested for them) and are hidden from their feed / explore. Idempotent. The now-hidden
|
||||
un-kept search/explore videos are reclaimed by the discovery-cleanup job in due course."""
|
||||
if db.get(Channel, channel_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
get_or_404(db, Channel, channel_id, what="channel")
|
||||
if not _is_blocked(db, user, channel_id):
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=channel_id))
|
||||
# Stop any active exploration of it so it can be cleaned up.
|
||||
@@ -706,7 +697,7 @@ def attach_tag(
|
||||
_user_subscription(db, user, channel_id)
|
||||
# Only the user's own tags may be attached as personal links (owned_or_404 → 404 if the tag
|
||||
# is missing OR belongs to someone else, without leaking which).
|
||||
owned_or_404(db, Tag, body.tag_id, user, detail="Unknown tag")
|
||||
owned_or_404(db, Tag, body.tag_id, user, what="tag")
|
||||
exists = db.execute(
|
||||
select(ChannelTag).where(
|
||||
ChannelTag.channel_id == channel_id,
|
||||
|
||||
@@ -10,6 +10,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.audit import AuditAction
|
||||
@@ -29,6 +30,14 @@ def _redact(v):
|
||||
return _URL_USERINFO.sub(r"\1***@", v) if isinstance(v, str) else v
|
||||
|
||||
|
||||
def _apply_runtime(db: Session, key: str) -> None:
|
||||
"""Some knobs change how THIS process behaves, not just what a later read returns — they have to
|
||||
take effect now, without a restart. One dispatch for both call sites (set and reset), so the next
|
||||
such key can't be wired into one of them and forgotten in the other."""
|
||||
if key == "log_level":
|
||||
logs.set_level(sysconfig.get_str(db, "log_level"))
|
||||
|
||||
|
||||
@router.get("")
|
||||
def get_config(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
return sysconfig.describe(db)
|
||||
@@ -80,6 +89,7 @@ def set_config(
|
||||
after=None if s.secret else {"value": _redact(new)},
|
||||
)
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
|
||||
@@ -104,6 +114,7 @@ def reset_config(
|
||||
before=None if s.secret else {"value": _redact(old)},
|
||||
)
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
"""The browser's log, sent while its user has a raised diagnostics level (E22 S5).
|
||||
|
||||
The client keeps its log in memory (frontend `lib/log.ts`). Only after the user consented to Részletes
|
||||
or Teljes in Settings → Diagnosztika does it stream that log here, in batches, and each batch lands in
|
||||
`log_events` beside the server's own rows: `source="web"`, the account from the SESSION (never from
|
||||
the body), and the request id each event carries, which is what joins a browser line to the server
|
||||
line of the same request.
|
||||
|
||||
Nothing from the body is trusted as-is:
|
||||
* only a JSON Content-Type is read, and the whole body is capped before it is parsed, and so are
|
||||
the number of events and each event's attributes;
|
||||
* the pace is capped per account, and so is what an account may store per hour;
|
||||
* everything is redacted AGAIN with the server's own rules — the client redacts at capture, but a
|
||||
modified client would not;
|
||||
* an event below the consented level is dropped (a Részletes consent does not store what only
|
||||
Teljes may carry), and so is the whole batch once the level has lapsed;
|
||||
* the timestamp is clamped to the consent window, so a row cannot dodge the retention job by
|
||||
claiming to come from the future.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy import insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import diagnostics, logs, logsink
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import LogEvent, User
|
||||
from app.ratelimit import RateLimiter
|
||||
|
||||
router = APIRouter(prefix="/api/diagnostics", tags=["diagnostics"])
|
||||
|
||||
BODY_MAX_BYTES = 512 * 1024
|
||||
EVENTS_MAX = 200
|
||||
ATTRS_MAX_BYTES = 16 * 1024
|
||||
# The client flushes every few seconds and when its buffer fills; this is room for a burst, not a
|
||||
# pace anything legitimate keeps up for a minute.
|
||||
_batches = RateLimiter(max_events=30, window_seconds=60)
|
||||
# What one account may STORE per hour. The pace cap alone still lets a self-consented account write
|
||||
# ~15 MiB a minute into the one shared database, kept for the whole retention window; a real session
|
||||
# at Teljes stores a few hundred kB an hour. Past the budget a batch is accepted and not stored (a
|
||||
# refusal would only make the client resend it), and one notice per hour says so.
|
||||
STORED_BYTES_PER_HOUR = 5 * 1024 * 1024
|
||||
_stored_bytes = RateLimiter(max_events=STORED_BYTES_PER_HOUR, window_seconds=3600)
|
||||
_budget_notices = RateLimiter(max_events=1, window_seconds=3600)
|
||||
|
||||
ClientLevel = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
|
||||
|
||||
class ClientEvent(BaseModel):
|
||||
ts: datetime
|
||||
level: ClientLevel
|
||||
event: str = Field(max_length=64, pattern=r"^[a-z][a-z0-9_]*(\.[a-z0-9_]+)*$")
|
||||
msg: str = Field(max_length=8_000)
|
||||
attrs: dict[str, Any] | None = None
|
||||
request_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$")
|
||||
tab_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,32}$")
|
||||
version: str | None = Field(default=None, max_length=64)
|
||||
|
||||
|
||||
class ClientEventsIn(BaseModel):
|
||||
events: list[ClientEvent] = Field(max_length=EVENTS_MAX)
|
||||
|
||||
|
||||
async def _capped_body(request: Request) -> bytes:
|
||||
"""The request body, refused the moment it passes BODY_MAX_BYTES while still arriving. Counted,
|
||||
not taken from Content-Length: a declared length can be absent (chunked) or wrong, and counting
|
||||
stops reading at the same point a declared-length check would have."""
|
||||
body = bytearray()
|
||||
async for chunk in request.stream():
|
||||
body += chunk
|
||||
if len(body) > BODY_MAX_BYTES:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
return bytes(body)
|
||||
|
||||
|
||||
async def _parsed_events(request: Request) -> ClientEventsIn:
|
||||
"""The body as the model — parsed only after the size cap, which is why this is a dependency and
|
||||
not a plain body parameter (FastAPI reads a declared body in full before anything else runs).
|
||||
|
||||
JSON only, by its Content-Type, as a declared body would be. Parsing any body as JSON would let a
|
||||
CORS-simple `text/plain` form post from a same-site page write events into a signed-in user's
|
||||
log; the app's own client always sends `application/json`, the unload beacon included."""
|
||||
media_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media_type != "application/json" and not media_type.endswith("+json"):
|
||||
raise HTTPException(status_code=415, detail={"code": "json_required"})
|
||||
raw = await _capped_body(request)
|
||||
try:
|
||||
return ClientEventsIn.model_validate_json(raw)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=422, detail=json.loads(exc.json())) from exc
|
||||
|
||||
|
||||
def _row(item: ClientEvent, user: User, now: datetime, window_start: datetime) -> dict[str, Any]:
|
||||
attrs = logs.redact(item.attrs) if item.attrs else None
|
||||
if attrs is not None:
|
||||
size = len(json.dumps(attrs, ensure_ascii=False, default=str).encode())
|
||||
if size > ATTRS_MAX_BYTES:
|
||||
attrs = {"attrs_dropped_bytes": size}
|
||||
if item.tab_id:
|
||||
attrs = {**(attrs or {}), "tab_id": item.tab_id}
|
||||
ts = item.ts if item.ts.tzinfo else item.ts.replace(tzinfo=timezone.utc)
|
||||
return logsink.to_row(
|
||||
{
|
||||
"ts": min(max(ts, window_start), now),
|
||||
"level": item.level,
|
||||
"sev": logs.SEVERITY[item.level],
|
||||
"source": "web",
|
||||
"logger": "web",
|
||||
"event": item.event,
|
||||
"msg": logs.redact_text(item.msg),
|
||||
"attrs": attrs,
|
||||
"request_id": item.request_id,
|
||||
"user_id": user.id,
|
||||
"version": item.version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/events")
|
||||
def ingest_events(
|
||||
# The account first: dependencies resolve in order, so an anonymous caller is refused before a
|
||||
# byte of its body is read.
|
||||
user: User = Depends(current_user),
|
||||
body: ClientEventsIn = Depends(_parsed_events),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
level = diagnostics.effective_level(user)
|
||||
if level == diagnostics.BASE:
|
||||
# Lapsed or withdrawn: the client stops streaming when it sees this.
|
||||
raise HTTPException(status_code=409, detail={"code": "diagnostics_off"})
|
||||
if not _batches.allow(str(user.id)):
|
||||
raise HTTPException(status_code=429, detail={"code": "rate_limited"})
|
||||
now = datetime.now(timezone.utc)
|
||||
window_start = now - timedelta(hours=diagnostics.ELEVATION_HOURS)
|
||||
floor = logs.SEVERITY[level]
|
||||
rows = [
|
||||
_row(item, user, now, window_start)
|
||||
for item in body.events
|
||||
if logs.SEVERITY[item.level] >= floor
|
||||
]
|
||||
if not rows:
|
||||
return {"ok": True, "stored": 0}
|
||||
size = sum(len(json.dumps(row, ensure_ascii=False, default=str).encode()) for row in rows)
|
||||
stored = len(rows)
|
||||
if not _stored_bytes.allow(str(user.id), size):
|
||||
stored = 0
|
||||
if not _budget_notices.allow(str(user.id)):
|
||||
return {"ok": True, "stored": 0}
|
||||
rows = [_budget_notice(user, now)]
|
||||
# Under the erasure guard until the commit: an account erased meanwhile gets no rows back.
|
||||
with logs.erasure_guard():
|
||||
db.execute(insert(LogEvent), logsink.without_erased(rows))
|
||||
db.commit()
|
||||
return {"ok": True, "stored": stored}
|
||||
|
||||
|
||||
def _budget_notice(user: User, now: datetime) -> dict[str, Any]:
|
||||
return logsink.to_row(
|
||||
{
|
||||
"ts": now,
|
||||
"level": "WARNING",
|
||||
"sev": logs.SEVERITY["WARNING"],
|
||||
"source": "web",
|
||||
"logger": "web",
|
||||
"event": "diagnostics.budget_spent",
|
||||
# What the budget does, exactly: it is a SLIDING hour, and a refused batch is not
|
||||
# charged to it, so smaller batches keep landing while there is room for them. Saying
|
||||
# "the rest of the hour is not kept" would be a row contradicting the rows beneath it.
|
||||
"msg": "The browser log passed its hourly storage budget; a batch bigger than what is left of it is not kept",
|
||||
"attrs": {"budget_bytes": STORED_BYTES_PER_HOUR},
|
||||
"user_id": user.id,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,197 @@
|
||||
"""E24 · the Home surface's HTTP shell.
|
||||
|
||||
Thin by construction: every decision — what counts as in progress, what fits a time budget, which
|
||||
tier the hero comes from — lives in `app/discover/`, so the scheduler, a future CLI or a test can ask
|
||||
the same questions without a request. This file validates, gates and serializes.
|
||||
|
||||
The consolidated `GET /api/discover` mirrors the Plex Discover route's one-payload shape (E5 S4): a
|
||||
landing page that fires four requests to answer one question would spend its whole budget on the
|
||||
thing it exists to remove.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user, require_human
|
||||
from app.db import get_db
|
||||
from app.discover import hero as hero_mod
|
||||
from app.discover import pins, rails
|
||||
from app.discover.budget import Budget, DEFAULT_BUDGET
|
||||
from app.models import ComfortPin, User
|
||||
from app.routes._common import owned_or_404
|
||||
|
||||
router = APIRouter(prefix="/api/discover", tags=["discover"])
|
||||
|
||||
|
||||
class PinIn(BaseModel):
|
||||
# From `pins.PIN_KINDS`, not a second hand-written list: the vocabulary, the schema columns and
|
||||
# this request contract have to move together, and re-typing them here is how they would not.
|
||||
kind: Literal[*pins.PIN_KINDS]
|
||||
# The id the rest of the API speaks for that kind: a video id, a channel id, a Plex rating_key,
|
||||
# or a playlist id. `pins.resolve_ref` does the translation to the row's internal key.
|
||||
ref: str
|
||||
|
||||
|
||||
class ReorderIn(BaseModel):
|
||||
ids: list[int]
|
||||
|
||||
|
||||
@router.get("")
|
||||
def get_discover(
|
||||
budget: Budget = DEFAULT_BUDGET,
|
||||
seed: int = 0,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Everything the Home page renders, in one pass.
|
||||
|
||||
`resume` and `shelf` are built once and handed to the hero, so the two rails the hero draws from
|
||||
are not queried twice for a single page load."""
|
||||
resume = rails.continue_watching(db, user)
|
||||
shelf = pins.shelf(db, user)
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf),
|
||||
"continue_watching": resume,
|
||||
"comfort": shelf,
|
||||
}
|
||||
|
||||
|
||||
# How many already-shown cards a reroll may carry, matched to the client's own rolling window
|
||||
# (`SEEN_WINDOW` in `Home.tsx`). A cheap sanity clamp, and deliberately no more than that: `exclude`
|
||||
# never reaches SQL — `hero.pick` uses it only as set membership over an already-bounded candidate
|
||||
# pool (the resume rail's `RAIL_LIMIT` rows, the shelf, `SURPRISE_DRAW` surprise draws) — so there is
|
||||
# no scan for a crafted URL to unbound, and the set is built in full before this slice anyway. The
|
||||
# comment that claimed otherwise was corrected in round 2; the request line's own length is what
|
||||
# actually bounds a hostile `exclude`.
|
||||
MAX_EXCLUDE = 32
|
||||
|
||||
|
||||
@router.get("/pick")
|
||||
def get_pick(
|
||||
budget: Budget = DEFAULT_BUDGET,
|
||||
seed: int = 0,
|
||||
exclude: str = "",
|
||||
start: int = 0,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The "Mást kérek" reroll: a hero on its own, for a new seed and/or a new budget.
|
||||
|
||||
One seed drives BOTH sources (see `discover/queries.py`) — otherwise a reroll would re-draw the
|
||||
Plex half while the YouTube half stood still, and the button would look broken half the time.
|
||||
|
||||
`exclude` is a comma-separated list of `<source>:<id>` keys the user has already been shown. It
|
||||
is what makes the button honest: a seed alone only moves the surprise tier, so a user with
|
||||
something half-finished got the same card back every press.
|
||||
|
||||
`start` is the press count: it rotates which tier answers first, so consecutive presses offer a
|
||||
different KIND of answer instead of walking down the resume rail. Taken modulo the tier count in
|
||||
`hero.pick`, so any integer is valid and no value can 422 a button."""
|
||||
skip = {p for p in exclude.split(",") if p}
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(
|
||||
db, user, budget, seed, exclude=set(list(skip)[:MAX_EXCLUDE]), start=start
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/pins")
|
||||
def list_pins(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The shelf on its own — for the shelf editor, which needs it without the rest of the page."""
|
||||
return {"items": pins.shelf(db, user)}
|
||||
|
||||
|
||||
@router.post("/pins")
|
||||
def add_pin(
|
||||
body: PinIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Pin something to the comfort shelf.
|
||||
|
||||
`require_human`, like every other personal-state write: the shared demo account must not be able
|
||||
to curate a communal shelf. Re-pinning something already on the shelf is a no-op that returns the
|
||||
existing pin rather than a 409 — the gesture is "make sure this is on my shelf", and the unique
|
||||
constraint is the mechanism, not the message."""
|
||||
resolved = pins.resolve_ref(db, user, body.kind, body.ref)
|
||||
if resolved is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
column, value = resolved
|
||||
|
||||
existing = db.execute(
|
||||
select(ComfortPin).where(
|
||||
ComfortPin.user_id == user.id,
|
||||
getattr(ComfortPin, column) == value,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return {"ok": True, "id": existing.id, "created": False}
|
||||
|
||||
pin = ComfortPin(user_id=user.id, position=pins.next_position(db, user), **{column: value})
|
||||
db.add(pin)
|
||||
try:
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
# A concurrent pin of the same thing from a second tab. The unique constraint did its job;
|
||||
# the user's intent ("be on my shelf") is satisfied either way, so report the winner rather
|
||||
# than a 500 the user can do nothing with.
|
||||
db.rollback()
|
||||
winner = db.execute(
|
||||
select(ComfortPin).where(
|
||||
ComfortPin.user_id == user.id,
|
||||
getattr(ComfortPin, column) == value,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if winner is None:
|
||||
# Not the unique constraint, then: the only other way in is the target row being pruned
|
||||
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
||||
# item), which trips the FK. Semantically that is the same "no such thing" the pre-check
|
||||
# returns above — re-raising handed the user a 500 they could do nothing with.
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
return {"ok": True, "id": winner.id, "created": False}
|
||||
return {"ok": True, "id": pin.id, "created": True}
|
||||
|
||||
|
||||
@router.delete("/pins/{pin_id}")
|
||||
def remove_pin(
|
||||
pin_id: int,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pin = owned_or_404(db, ComfortPin, pin_id, user, what="pin")
|
||||
db.delete(pin)
|
||||
db.commit()
|
||||
return {"deleted": pin_id}
|
||||
|
||||
|
||||
@router.post("/pins/reorder")
|
||||
def reorder_pins(
|
||||
body: ReorderIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Rewrite the shelf order from a full list of pin ids.
|
||||
|
||||
Whole-list rather than a move-one delta: the shelf is short, and a full rewrite cannot leave two
|
||||
tiles claiming the same slot the way an interrupted sequence of single moves can. Ids that are not
|
||||
this user's are ignored rather than 404'd — a stale tab replaying an order that includes a pin
|
||||
deleted meanwhile should reorder what still exists, not fail wholesale."""
|
||||
owned = {p.id for p in pins.list_pins(db, user)}
|
||||
for position, pin_id in enumerate(i for i in body.ids if i in owned):
|
||||
db.execute(
|
||||
update(ComfortPin)
|
||||
.where(ComfortPin.id == pin_id, ComfortPin.user_id == user.id)
|
||||
.values(position=position)
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
@@ -38,6 +38,7 @@ from app.models import (
|
||||
MediaAsset,
|
||||
User,
|
||||
)
|
||||
from app.routes._common import get_or_404, not_found
|
||||
from app.security import hash_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
@@ -106,6 +107,7 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"phase": job.phase,
|
||||
"error": job.error,
|
||||
"error_code": job.error_code,
|
||||
"error_detail": job.error_detail,
|
||||
"queue_pos": job.queue_pos,
|
||||
"created_at": iso(job.created_at),
|
||||
"source_kind": job.source_kind,
|
||||
@@ -171,6 +173,32 @@ def _own_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
return job
|
||||
|
||||
|
||||
def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""`_own_job` for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
||||
or writes the status those decide on (pause). The job is re-read under a row lock, so two
|
||||
overlapping requests for one job run one after the other and the second decides on the first's
|
||||
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||
just released back into a holder; either can free a file another job still holds.
|
||||
|
||||
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||
(`service.finalize_if_running`), so the two can never deadlock."""
|
||||
while True:
|
||||
asset_id = _own_job(db, user, job_id).asset_id
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
if job.asset_id == asset_id:
|
||||
return job
|
||||
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||
|
||||
|
||||
# --- profiles ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -604,7 +632,7 @@ def update_download_meta(
|
||||
def pause_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.pause(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -613,7 +641,7 @@ def pause_download(
|
||||
def resume_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.resume(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -628,12 +656,13 @@ def redownload_download(
|
||||
resolves to the newly-downloaded file — a re-download never breaks a link you already sent out
|
||||
(only a *failed* re-download leaves the job in `error`, which is the intended broken-link case).
|
||||
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the job
|
||||
id + current asset id) rather than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the new
|
||||
asset's own id, and the user's cookie salt when one applies — `credentials.rebuild_salt`) rather
|
||||
than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
under the current format/naming rules — never silently reuses a shared cached file — and (b)
|
||||
never deletes a file another job/user still shares: the old asset is only removed if THIS job
|
||||
was its last holder."""
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
if job.job_kind in ("edit", "upload"):
|
||||
# Neither has a re-fetchable source: an editor clip is derived locally, an upload IS the
|
||||
# source. Re-running yt-dlp against source_kind='upload'/'edit' would just fail.
|
||||
@@ -648,7 +677,7 @@ def redownload_download(
|
||||
def cancel_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.cancel(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -657,7 +686,7 @@ def cancel_download(
|
||||
def delete_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.delete(db, job)
|
||||
return {"deleted": job_id}
|
||||
|
||||
@@ -801,7 +830,7 @@ def share_download(
|
||||
# Uniform "no user" for a missing OR non-messageable (demo/suspended/deactivated) target, so
|
||||
# sharing can't be used to probe which addresses exist or to reach a suspended account.
|
||||
if not is_messageable_user(recipient):
|
||||
raise HTTPException(status_code=404, detail="No user with that email.")
|
||||
raise not_found("recipient")
|
||||
if recipient.id == user.id:
|
||||
raise HTTPException(status_code=400, detail="That's already your download.")
|
||||
exists = db.execute(
|
||||
@@ -1091,8 +1120,7 @@ def admin_set_quota(
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if db.get(User, user_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown user")
|
||||
get_or_404(db, User, user_id, what="user")
|
||||
cur = quota.resolve(db, user_id)
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is None:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -22,12 +22,14 @@ from app.feedquery import (
|
||||
sort_keys,
|
||||
)
|
||||
from app.models import (
|
||||
POSITION_MAX_S,
|
||||
ChannelTag,
|
||||
Tag,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.routes._common import get_or_404
|
||||
from app.sync.videos import parse_iso8601_duration
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
@@ -234,8 +236,7 @@ def set_video_state(
|
||||
status = body.status
|
||||
if status not in VALID_STATES:
|
||||
raise HTTPException(status_code=400, detail=f"status must be one of {VALID_STATES}")
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
@@ -303,11 +304,16 @@ def clear_video_state(
|
||||
|
||||
|
||||
class VideoProgressIn(BaseModel):
|
||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||
# one into a 500.
|
||||
model_config = ConfigDict(allow_inf_nan=False)
|
||||
|
||||
# Accept a float and truncate (below) — the player may post a raw currentTime (a float), which
|
||||
# the old `int(payload.get(...))` silently truncated. A plain `int` field would 422 that on the
|
||||
# 10s progress hot path, losing the resume position; `float` keeps the old tolerance.
|
||||
position_seconds: float = 0
|
||||
duration_seconds: float = 0
|
||||
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
||||
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
|
||||
|
||||
@router.post("/videos/{video_id}/progress")
|
||||
@@ -326,8 +332,7 @@ def set_video_progress(
|
||||
if position < 0:
|
||||
position = 0
|
||||
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
# Decide whether this position is worth keeping (else clear it).
|
||||
near_end = duration > 0 and position > duration - FINISH_MARGIN_SECONDS
|
||||
|
||||
+57
-13
@@ -1,5 +1,7 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, RootModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -7,6 +9,7 @@ from app.auth import (
|
||||
count_admins,
|
||||
count_pending_activations,
|
||||
current_user,
|
||||
drop_from_wallet,
|
||||
google_enabled,
|
||||
has_read_scope,
|
||||
has_write_scope,
|
||||
@@ -14,7 +17,9 @@ from app.auth import (
|
||||
optional_current_user,
|
||||
purge_user,
|
||||
)
|
||||
from app import diagnostics
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import sysconfig
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
@@ -65,24 +70,26 @@ def switch_account(
|
||||
target = body.user_id
|
||||
if target not in (request.session.get("account_ids") or []):
|
||||
raise HTTPException(status_code=403, detail="Not an account you've signed into here.")
|
||||
# An account `_session_user` would refuse — gone, deactivated, suspended, or its session revoked — is
|
||||
# refused here too, and dropped from the switch list the way `_session_user` drops it. Made the session
|
||||
# default instead, the next request's current_user would refuse it as the DEFAULT and clear the whole
|
||||
# wallet session, logging out every account here. (B-core-9)
|
||||
u = db.get(User, target)
|
||||
if u is None:
|
||||
drop_from_wallet(request.session, target)
|
||||
raise HTTPException(status_code=404, detail="That account no longer exists.")
|
||||
if not is_allowed(db, u.email):
|
||||
raise HTTPException(status_code=403, detail="That account no longer has access.")
|
||||
if not u.is_active:
|
||||
drop_from_wallet(request.session, target)
|
||||
raise HTTPException(status_code=403, detail="That account no longer has access.")
|
||||
if u.is_suspended:
|
||||
# Don't make a suspended account the active one — the next request's current_user would
|
||||
# see the suspension and clear the WHOLE wallet session, logging out every account here.
|
||||
drop_from_wallet(request.session, target)
|
||||
raise HTTPException(status_code=403, detail="That account is suspended.")
|
||||
# Same reasoning for a REVOKED session (a password reset/change or "log out everywhere" bumped the
|
||||
# account's epoch): making it the session default would trip current_user's epoch check on the next
|
||||
# request and clear the whole wallet. Mirror the suspend guard — drop the stale account from the
|
||||
# switch list and refuse, rather than logging everyone out. (B-core-9)
|
||||
# Revoked: a password reset/change or "log out everywhere" bumped the account's epoch.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(target), 0)) != (u.session_epoch or 0):
|
||||
request.session["account_ids"] = [
|
||||
a for a in (request.session.get("account_ids") or []) if a != target
|
||||
]
|
||||
drop_from_wallet(request.session, target)
|
||||
raise HTTPException(
|
||||
status_code=403, detail="That account was signed out elsewhere — sign in again."
|
||||
)
|
||||
@@ -144,6 +151,9 @@ def get_me(
|
||||
"pending_activations": pending_activations,
|
||||
"email_configured": email_configured,
|
||||
"preferences": user.preferences or {},
|
||||
# The consented diagnostics level in force and when it lapses (E22 S5). The client applies
|
||||
# it to its own log and streams that log while it is raised.
|
||||
"diagnostics": diagnostics.state(db, user),
|
||||
}
|
||||
|
||||
|
||||
@@ -159,8 +169,13 @@ def delete_my_account(
|
||||
which all cascade on the users row. Quota-audit events are kept but anonymised (FK SET NULL).
|
||||
The shared demo account can't be deleted, and the last remaining admin can't delete itself
|
||||
(that would lock everyone out of the admin surfaces)."""
|
||||
# Structured, like the other three demo refusals: `req` leaves 403 to the caller and
|
||||
# `notifyActionError` discards a plain-string detail in favour of the caller's translated
|
||||
# fallback, so a hand-written English sentence here reaches a Hungarian UI as a generic
|
||||
# "couldn't do that" — or, on this affordance, as nothing at all. `reason` distinguishes it
|
||||
# from the other sites' bare `{code: "demo"}` so the UI can say what specifically is refused.
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="The shared demo account can't be deleted.")
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "delete_account"})
|
||||
if user.role == "admin" and count_admins(db) <= 1:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
@@ -180,16 +195,45 @@ def delete_my_account(
|
||||
return {"deleted": True}
|
||||
|
||||
|
||||
class PreferencesIn(RootModel[dict[str, Any]]):
|
||||
"""Free-form UI preferences — a JSON object of any keys (theme, filters, …), merged into the
|
||||
stored ones. A model rather than a bare `dict` parameter so a non-object body is a 422 at the
|
||||
boundary like every other request body."""
|
||||
|
||||
|
||||
@router.put("/preferences")
|
||||
def update_preferences(
|
||||
preferences: dict,
|
||||
preferences: PreferencesIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# Merge so partial updates don't wipe other keys.
|
||||
merged = dict(user.preferences or {})
|
||||
merged.update(preferences)
|
||||
merged.update(preferences.root)
|
||||
user.preferences = merged
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return {"preferences": merged}
|
||||
|
||||
|
||||
class DiagnosticsIn(BaseModel):
|
||||
level: diagnostics.DiagnosticsLevel
|
||||
|
||||
|
||||
@router.put("/diagnostics")
|
||||
def update_diagnostics(
|
||||
body: DiagnosticsIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Raise the diagnostics level (the user consented in Settings → Diagnosztika) or return to the
|
||||
base. The expiry is the server's: a raised level always lapses after
|
||||
`diagnostics.ELEVATION_HOURS`, whatever the client asks. The shared demo account cannot raise
|
||||
it — its log would interleave every visitor's session under one account."""
|
||||
if user.is_demo and body.level != diagnostics.BASE:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "diagnostics"})
|
||||
diagnostics.set_level(user, body.level)
|
||||
db.commit()
|
||||
# The rest of THIS request, and the process's elevation registry, follow the new level now.
|
||||
logs.set_user(user.id, diagnostics.effective_level(user), user.diagnostics_until)
|
||||
return {"ok": True, "diagnostics": diagnostics.state(db, user)}
|
||||
|
||||
@@ -46,7 +46,7 @@ WELCOME = {
|
||||
"This is your private message inbox. Conversations with other members are end-to-end "
|
||||
"encrypted — only you and the person you're chatting with can read them, not even an "
|
||||
"admin. To start messaging someone, set up secure messaging with a passphrase first.\n\n"
|
||||
"Enjoy your feed!"
|
||||
"Enjoy your Feed!"
|
||||
),
|
||||
"hu": (
|
||||
"Üdv a Siftlode-on! 👋\n\n"
|
||||
@@ -423,9 +423,10 @@ async def messages_ws(ws: WebSocket) -> None:
|
||||
"""Live push channel: authenticated by the session cookie, registers the connection so
|
||||
sent messages reach this user's open tabs instantly. We don't consume client→server frames
|
||||
(sending goes through POST /api/messages); the receive loop only detects disconnect."""
|
||||
# Which signed-in account this socket acts as — the SHARED per-tab resolution (wallet-gated
|
||||
# ?account= override, session default otherwise). A browser WebSocket can't send the
|
||||
# X-Siftlode-Account header, so resolved_user_id falls through to the ?account= query param.
|
||||
# Which signed-in account this socket acts as — the SHARED per-tab resolution (a wallet-gated
|
||||
# ?account=, the session default when none is named, nobody for an account no longer signed in
|
||||
# here). A browser WebSocket can't send the X-Siftlode-Account header, so resolved_user_id falls
|
||||
# through to the ?account= query param.
|
||||
uid, _ = resolved_user_id(ws)
|
||||
if not uid:
|
||||
await ws.close(code=1008)
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.sync.playlists import (
|
||||
sync_user_playlists,
|
||||
)
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import get_or_404, owned_or_404
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
|
||||
router = APIRouter(prefix="/api/playlists", tags=["playlists"])
|
||||
@@ -48,10 +49,13 @@ def _check_pull_cooldown(user: User) -> None:
|
||||
|
||||
|
||||
def _own_playlist(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
pl = db.get(Playlist, playlist_id)
|
||||
if pl is None or pl.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown playlist")
|
||||
return pl
|
||||
return owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
|
||||
|
||||
def _refused(status: int, reason: str) -> HTTPException:
|
||||
"""A playlist action the server refuses, as a reason the UI translates
|
||||
(`playlists.errors.<reason>`) — a bare English detail reached the Hungarian UI verbatim."""
|
||||
return HTTPException(status_code=status, detail={"code": "playlist", "reason": reason})
|
||||
|
||||
|
||||
def _touch(pl: Playlist) -> None:
|
||||
@@ -256,7 +260,7 @@ def create_playlist(
|
||||
) -> dict:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name is required")
|
||||
raise _refused(400, "name_required")
|
||||
pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id))
|
||||
db.add(pl)
|
||||
db.commit()
|
||||
@@ -288,8 +292,7 @@ def add_watch_later(
|
||||
) -> dict:
|
||||
"""Bookmark shortcut: add a video to the built-in Watch later playlist."""
|
||||
video_id = body.video_id
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
pl = _watch_later(db, user)
|
||||
_add_item(db, pl, video_id, mark_dirty=False)
|
||||
return {"saved": True, "playlist_id": pl.id}
|
||||
@@ -317,10 +320,7 @@ def sync_youtube(
|
||||
"""Pull the user's YouTube playlists into local mirrors now (read-only direction).
|
||||
Also runs automatically on the scheduler; this is the manual 'sync now'."""
|
||||
if not has_read_scope(user):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Connect YouTube (read access) to sync your playlists.",
|
||||
)
|
||||
raise _refused(403, "youtube_read_scope")
|
||||
_check_pull_cooldown(user)
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return sync_user_playlists(db, user)
|
||||
@@ -337,18 +337,16 @@ def revert_youtube(
|
||||
once the in-session undo history is gone."""
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
if not pl.yt_playlist_id:
|
||||
raise HTTPException(status_code=400, detail="This playlist isn't linked to YouTube.")
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_read_scope(user):
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Connect YouTube (read access) to sync your playlists."
|
||||
)
|
||||
raise _refused(403, "youtube_read_scope")
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return repull_playlist(db, user, pl)
|
||||
except YouTubeError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=422, detail="Couldn't refresh from YouTube.")
|
||||
raise _refused(422, "youtube_failed")
|
||||
|
||||
|
||||
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
@@ -356,9 +354,7 @@ def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
Watch later and read-only YouTube mirrors are never pushable."""
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
if pl.kind == "watch_later":
|
||||
raise HTTPException(
|
||||
status_code=400, detail="The Watch later list can't be sent to YouTube."
|
||||
)
|
||||
raise _refused(400, "watch_later_not_pushable")
|
||||
return pl
|
||||
|
||||
|
||||
@@ -376,7 +372,7 @@ def push_plan(
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
plan = plan_merge(db, user, pl)
|
||||
except YouTubeError:
|
||||
raise HTTPException(status_code=422, detail="Couldn't read the playlist on YouTube.")
|
||||
raise _refused(422, "youtube_failed")
|
||||
plan["remaining_today"] = quota.remaining_today(db)
|
||||
plan["affordable"] = plan["units_estimate"] <= plan["remaining_today"]
|
||||
return plan
|
||||
@@ -406,14 +402,11 @@ def push(
|
||||
)
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="Not enough YouTube quota left today for this sync.",
|
||||
)
|
||||
raise _refused(429, "quota_exceeded")
|
||||
return merge_playlist(db, user, pl, live=live)
|
||||
except YouTubeError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=422, detail="YouTube playlist sync failed.")
|
||||
raise _refused(422, "youtube_failed")
|
||||
|
||||
|
||||
@router.get("/{playlist_id}")
|
||||
@@ -458,7 +451,7 @@ def rename_playlist(
|
||||
if "name" in body.model_fields_set:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name cannot be empty")
|
||||
raise _refused(400, "name_required")
|
||||
if name != pl.name:
|
||||
pl.name = name
|
||||
recompute_dirty(db, pl)
|
||||
@@ -493,18 +486,18 @@ def delete_playlist(
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
if pl.kind == "watch_later":
|
||||
raise HTTPException(status_code=400, detail="The Watch later list can't be deleted.")
|
||||
raise _refused(400, "watch_later_undeletable")
|
||||
if on_youtube:
|
||||
if not pl.yt_playlist_id:
|
||||
raise HTTPException(status_code=400, detail="This playlist isn't on YouTube.")
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_write_scope(user):
|
||||
raise HTTPException(status_code=403, detail="Enable YouTube editing in Settings first.")
|
||||
raise _refused(403, "youtube_editing_off")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt.delete_playlist(pl.yt_playlist_id)
|
||||
except YouTubeError:
|
||||
raise HTTPException(status_code=422, detail="YouTube delete failed.")
|
||||
raise _refused(422, "youtube_failed")
|
||||
db.delete(pl) # items cascade
|
||||
db.commit()
|
||||
return {"deleted": playlist_id}
|
||||
@@ -519,8 +512,7 @@ def add_item(
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
video_id = body.video_id
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
_add_item(db, pl, video_id, mark_dirty=True)
|
||||
return {"playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
@@ -557,7 +549,9 @@ def move_item(
|
||||
case. Idempotent: if the video is already in the target, the source row is simply dropped
|
||||
(moving into a list that already has it is a removal, not a duplicate)."""
|
||||
src = _own_playlist(db, user, playlist_id)
|
||||
dst = _own_playlist(db, user, body.target_playlist_id)
|
||||
# Its own reason: "this playlist doesn't exist any more" about a TARGET that is gone was read as
|
||||
# the list on screen — which is still there.
|
||||
dst = owned_or_404(db, Playlist, body.target_playlist_id, user, what="move_target")
|
||||
if src.id == dst.id:
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "playlist_move", "reason": "same_playlist"}
|
||||
|
||||
@@ -3,13 +3,9 @@
|
||||
import logging
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import (
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
)
|
||||
from app.models import PlexItem
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
@@ -25,21 +21,3 @@ def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
||||
def _norm_lang(lang: str | None) -> str:
|
||||
"""The overlay has an EN base + an optional HU twin; anything other than `hu` serves EN."""
|
||||
return "hu" if lang == "hu" else "en"
|
||||
|
||||
|
||||
def _content_libs(db: Session) -> tuple[list[int], list[int]]:
|
||||
"""The BROWSABLE content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
|
||||
source of the browse scope. Excludes HU overlay libraries (`overlay_of` set): those are never
|
||||
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows (E5 S5)."""
|
||||
libs = (
|
||||
db.execute(
|
||||
select(PlexLibrary).where(
|
||||
PlexLibrary.enabled.is_(True), PlexLibrary.overlay_of.is_(None)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
movie_lib_ids = [lb.id for lb in libs if lb.kind == "movie"]
|
||||
show_lib_ids = [lb.id for lb in libs if lb.kind == "show"]
|
||||
return movie_lib_ids, show_lib_ids
|
||||
|
||||
@@ -34,6 +34,7 @@ from app.plex.query import (
|
||||
apply_meta_filters,
|
||||
apply_movie_state_filter,
|
||||
apply_show_state_filter,
|
||||
content_libs,
|
||||
field_recheck,
|
||||
genres_col,
|
||||
meta_filter_conds,
|
||||
@@ -47,12 +48,28 @@ from app.plex.query import (
|
||||
vector_col,
|
||||
vector_match,
|
||||
)
|
||||
from app.routes.plex._shared import _content_libs, _norm_lang
|
||||
from app.routes.plex._shared import _norm_lang
|
||||
from app.utils import to_tsquery_str
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
|
||||
def sort_column(u, sort: str):
|
||||
"""The browse grid's sort key -> orderable column on the union subquery. Module-level (not inline
|
||||
in the handler) so a test can pin its vocabulary to `PLEX_SORT_KEYS` — a key the Sort control
|
||||
offers but this cannot order would quietly fall back to `added_at` and look like a broken control.
|
||||
An unknown key (a stale bookmark, a hand-edited URL) deliberately keeps that fallback."""
|
||||
return {
|
||||
"added": u.c.added_at,
|
||||
"title": func.lower(u.c.title),
|
||||
"year": u.c.year,
|
||||
"rating": u.c.rating,
|
||||
"duration": u.c.duration_s,
|
||||
"release": u.c.rel,
|
||||
"lastwatched": u.c.last_watched,
|
||||
}.get(sort, u.c.added_at)
|
||||
|
||||
|
||||
@router.get("/libraries")
|
||||
def list_libraries(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""The mirrored libraries the user can browse (drives the Plex scope selector). HU overlay
|
||||
@@ -109,7 +126,7 @@ def unified_library(
|
||||
offset = max(0, offset)
|
||||
lang = _norm_lang(lang)
|
||||
p = f.as_dict()
|
||||
movie_lib_ids, show_lib_ids = _content_libs(db)
|
||||
movie_lib_ids, show_lib_ids = content_libs(db)
|
||||
want_movies = scope in ("movie", "both") and bool(movie_lib_ids)
|
||||
want_shows = scope in ("show", "both") and bool(show_lib_ids)
|
||||
|
||||
@@ -220,14 +237,7 @@ def unified_library(
|
||||
if tsq is not None:
|
||||
order = [u.c.rank.desc()]
|
||||
else:
|
||||
col = {
|
||||
"title": func.lower(u.c.title),
|
||||
"year": u.c.year,
|
||||
"rating": u.c.rating,
|
||||
"duration": u.c.duration_s,
|
||||
"release": u.c.rel,
|
||||
"lastwatched": u.c.last_watched,
|
||||
}.get(sort, u.c.added_at)
|
||||
col = sort_column(u, sort)
|
||||
order = [(col.asc() if sort_dir == "asc" else col.desc()).nullslast()]
|
||||
order.append(u.c.rk.desc())
|
||||
rows = db.execute(select(u).order_by(*order).offset(offset).limit(limit)).all()
|
||||
@@ -328,7 +338,7 @@ def facets(
|
||||
"duration_min": None,
|
||||
"duration_max": None,
|
||||
}
|
||||
all_movie_ids, all_show_ids = _content_libs(db)
|
||||
all_movie_ids, all_show_ids = content_libs(db)
|
||||
movie_ids = all_movie_ids if scope in ("movie", "both") else []
|
||||
show_ids = all_show_ids if scope in ("show", "both") else []
|
||||
if not movie_ids and not show_ids:
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
"""Plex collections: browse mirrored collections + the admin manual-collection editor."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import date, datetime
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
@@ -12,6 +16,7 @@ from app.models import (
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
PlexShow,
|
||||
PlexState,
|
||||
User,
|
||||
)
|
||||
from app.plex import rollup
|
||||
@@ -19,8 +24,7 @@ from app.plex import sync as plex_sync
|
||||
from app.plex.cards import movie_card, show_card
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.plex.query import like_escape, runtime_by_key, top_art_keys
|
||||
from app.routes.plex._shared import _content_libs
|
||||
from app.plex.query import content_libs, like_escape, runtime_by_key, top_art_keys
|
||||
|
||||
# The most-restrictive → least ordering used to pick a collection's effective content rating from its
|
||||
# members (Plex shows one cert on the collection; we don't store it, so derive it).
|
||||
@@ -84,13 +88,20 @@ def _collection_card(c: PlexCollection) -> dict:
|
||||
def collections(
|
||||
library: str | None = None,
|
||||
q: str | None = None,
|
||||
keys: str | None = None,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Mirrored collections as cards (most-populated first), optionally name-filtered by `q`. With a
|
||||
`library` plex_key → that one library's collections (used by the collection editor). WITHOUT one →
|
||||
the UNION across all enabled libraries (the unified-scope sidebar picker, which has no single
|
||||
library selected)."""
|
||||
library selected).
|
||||
|
||||
`keys` (CSV of rating_keys) narrows to just those. It exists because a Plex link carries the
|
||||
collection FILTER as bare rating_keys — nothing in the URL says what "17045" is called — so the
|
||||
sidebar token and the applied-filter chip had nothing to render but the raw id. Resolving them
|
||||
through this endpoint costs one small row each; pulling the whole list to find two titles is
|
||||
~200 KB."""
|
||||
if library is not None:
|
||||
lib = (
|
||||
db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == str(library)))
|
||||
@@ -114,6 +125,13 @@ def collections(
|
||||
term = (q or "").strip()
|
||||
if term:
|
||||
query = query.where(PlexCollection.title.ilike(f"%{like_escape(term)}%"))
|
||||
if keys is not None:
|
||||
wanted = [k for k in (keys or "").split(",") if k]
|
||||
# An explicit but empty `keys=` asks for nothing — answer nothing, rather than falling through
|
||||
# to "every collection" (which is what an unguarded empty IN () would have to become).
|
||||
if not wanted:
|
||||
return {"collections": []}
|
||||
query = query.where(PlexCollection.rating_key.in_(wanted))
|
||||
rows = (
|
||||
db.execute(
|
||||
query.order_by(
|
||||
@@ -128,7 +146,7 @@ def collections(
|
||||
# members (a collection lives in one library, so exactly one side has art) — a show collection would
|
||||
# otherwise render blank; the collection's own poster is the next fallback, then the colour hash.
|
||||
# Runtime is movie-only (shows have no single duration).
|
||||
movie_ids, show_ids = _content_libs(db)
|
||||
movie_ids, show_ids = content_libs(db)
|
||||
keys = [c.rating_key for c in rows]
|
||||
art_m = top_art_keys(db, PlexItem, movie_ids, PlexItem.collection_keys, keys, per=2)
|
||||
art_s = top_art_keys(db, PlexShow, show_ids, PlexShow.collection_keys, keys, per=2)
|
||||
@@ -150,14 +168,69 @@ def collections(
|
||||
}
|
||||
|
||||
|
||||
class Member(NamedTuple):
|
||||
"""One collection member flattened to everything the Sort control can order by. The page's two
|
||||
legs are different tables (movies and shows), so there is no single query to ORDER BY — the
|
||||
ordering is a pure list sort over these, which is also what makes it unit-testable."""
|
||||
|
||||
kind: str # "movie" | "show"
|
||||
obj: Any # PlexItem | PlexShow, carried through to the card serializer
|
||||
title: str # lower-cased; never NULL (the column is not nullable)
|
||||
year: int | None
|
||||
rating: float | None
|
||||
added: datetime | None
|
||||
release: date | None
|
||||
duration: int | None # movies only — a show has no single runtime
|
||||
last_watched: datetime | None
|
||||
|
||||
|
||||
# Sort key -> what to read off a Member. Pinned to `PLEX_SORT_KEYS` by a test: the collection page
|
||||
# shows the SAME Sort control as the browse grid, so a key this map is missing would be an option
|
||||
# that silently does nothing here.
|
||||
MEMBER_SORT_KEYS: dict[str, Callable[[Member], Any]] = {
|
||||
"added": lambda m: m.added,
|
||||
"lastwatched": lambda m: m.last_watched,
|
||||
"release": lambda m: m.release,
|
||||
"year": lambda m: m.year,
|
||||
"rating": lambda m: m.rating,
|
||||
"duration": lambda m: m.duration,
|
||||
"title": lambda m: m.title,
|
||||
}
|
||||
|
||||
# What the page ordered by before it had a control, and still its default: a film series wants to be
|
||||
# read in release order, not by whenever Plex happened to ingest it.
|
||||
DEFAULT_MEMBER_SORT = "year"
|
||||
|
||||
|
||||
def sort_members(members: list[Member], sort: str, sort_dir: str) -> list[Member]:
|
||||
"""Order a collection's members by one of `PLEX_SORT_KEYS`.
|
||||
|
||||
Two properties worth stating because they are easy to get wrong with a plain `reverse=True`:
|
||||
members with no value for the key sink to the END in BOTH directions (matching the browse grid's
|
||||
`nullslast()` — a movie with no rating is not "the worst rated"), and ties always resolve
|
||||
title-ascending, so flipping the direction reverses the ranking rather than the alphabet inside
|
||||
it. An unknown key falls back to the page's default instead of returning an arbitrary order."""
|
||||
key = MEMBER_SORT_KEYS.get(sort, MEMBER_SORT_KEYS[DEFAULT_MEMBER_SORT])
|
||||
ordered = sorted(members, key=lambda m: m.title) # stable secondary, applied first
|
||||
have = [m for m in ordered if key(m) is not None]
|
||||
missing = [m for m in ordered if key(m) is None]
|
||||
have.sort(key=key, reverse=sort_dir != "asc")
|
||||
return have + missing
|
||||
|
||||
|
||||
@router.get("/collections/{rating_key}")
|
||||
def collection_detail(
|
||||
rating_key: str,
|
||||
sort: str = DEFAULT_MEMBER_SORT,
|
||||
sort_dir: str = "asc",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""One collection's hero data + its members (movies + shows), ordered by release year then title —
|
||||
the Collection info page. Read-only here; edits go through the admin mutation endpoints below."""
|
||||
"""One collection's hero data + its members (movies + shows) in the requested order — the
|
||||
Collection info page. The sort vocabulary is the browse grid's (`PLEX_SORT_KEYS`) so the same Sort
|
||||
control means the same thing on both surfaces; the default is release year ascending, which is
|
||||
what this page did before it had a control. Read-only here; edits go through the admin mutation
|
||||
endpoints below."""
|
||||
col = by_rating_key(db, PlexCollection, rating_key)
|
||||
if col is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown collection")
|
||||
@@ -180,19 +253,63 @@ def collection_detail(
|
||||
.all()
|
||||
)
|
||||
mstate = rollup.item_state_map(db, user.id, [m.id for m in movies])
|
||||
sstate = rollup.state_map(db, user.id, [s.id for s in shows])
|
||||
# Merge the two type-homogeneous lists into ONE member list ordered by (year, title).
|
||||
tagged = [(m.year, (m.title or "").lower(), "movie", m) for m in movies] + [
|
||||
(s.year, (s.title or "").lower(), "show", s) for s in shows
|
||||
# ONE rollup pass over the show members, for both the aggregate badge status and `resumed_at` —
|
||||
# the browse grid's "last watched" for a show (the newest resume/watched across its episodes). The
|
||||
# cheaper `rollup.state_map` gives only the status, which would leave `lastwatched` unorderable on
|
||||
# a show collection while the control still offered it.
|
||||
show_rows = (
|
||||
db.execute(
|
||||
rollup.agg_subquery(db, user.id, include_resumed=True).where(
|
||||
PlexItem.show_id.in_([s.id for s in shows])
|
||||
)
|
||||
).all()
|
||||
if shows
|
||||
else []
|
||||
)
|
||||
sstate = {r.show_id: rollup.show_status(r.total, r.watched, r.inprog) for r in show_rows}
|
||||
sresumed = {r.show_id: r.resumed_at for r in show_rows}
|
||||
|
||||
def _movie_last_watched(st: PlexState | None) -> datetime | None:
|
||||
# Same definition as the browse grid's: when it was last resumed, else when it was marked
|
||||
# watched (coalesce(progress_updated_at, watched_at)).
|
||||
return (st.progress_updated_at or st.watched_at) if st else None
|
||||
|
||||
# Merge the two type-homogeneous lists into ONE member list, then order it.
|
||||
members = [
|
||||
Member(
|
||||
kind="movie",
|
||||
obj=m,
|
||||
title=(m.title or "").lower(),
|
||||
year=m.year,
|
||||
rating=m.rating,
|
||||
added=m.added_at,
|
||||
release=m.originally_available_at,
|
||||
duration=m.duration_s,
|
||||
last_watched=_movie_last_watched(mstate.get(m.id)),
|
||||
)
|
||||
for m in movies
|
||||
] + [
|
||||
Member(
|
||||
kind="show",
|
||||
obj=sh,
|
||||
title=(sh.title or "").lower(),
|
||||
year=sh.year,
|
||||
rating=sh.rating,
|
||||
added=sh.added_at,
|
||||
release=sh.originally_available_at,
|
||||
duration=None,
|
||||
last_watched=sresumed.get(sh.id),
|
||||
)
|
||||
for sh in shows
|
||||
]
|
||||
tagged.sort(key=lambda t: (t[0] is None, t[0] or 0, t[1]))
|
||||
members = sort_members(members, sort, sort_dir)
|
||||
items = [
|
||||
movie_card(o, mstate.get(o.id))
|
||||
if kind == "movie"
|
||||
else show_card(o, sstate.get(o.id, "new"))
|
||||
for _, _, kind, o in tagged
|
||||
movie_card(m.obj, mstate.get(m.obj.id))
|
||||
if m.kind == "movie"
|
||||
else show_card(m.obj, sstate.get(m.obj.id, "new"))
|
||||
for m in members
|
||||
]
|
||||
years = [y for y, _, _, _ in tagged if y is not None]
|
||||
years = [m.year for m in members if m.year is not None]
|
||||
certs = [m.content_rating for m in movies] + [s.content_rating for s in shows]
|
||||
return {
|
||||
**_collection_card(col),
|
||||
@@ -230,7 +347,9 @@ def _reject_duplicate_name(
|
||||
.first()
|
||||
)
|
||||
if dup is not None and dup.rating_key != exclude_rk:
|
||||
raise HTTPException(status_code=409, detail="A collection with this name already exists")
|
||||
# Structured, so the bilingual UI says it in the reader's language (it was toasted in
|
||||
# Hungarian beside an English error dialog saying the same thing).
|
||||
raise HTTPException(status_code=409, detail={"code": "collection", "reason": "name_taken"})
|
||||
|
||||
|
||||
def _write_or_pending_error(wrote: bool, e: Exception) -> HTTPException:
|
||||
|
||||
@@ -27,12 +27,13 @@ from app.plex.query import (
|
||||
LibraryFilters,
|
||||
apply_meta_filters,
|
||||
apply_show_state_filter,
|
||||
content_libs,
|
||||
genres_col,
|
||||
meta_filter_conds,
|
||||
top_art_keys,
|
||||
)
|
||||
from app.plex.tmdb import resolve_recs
|
||||
from app.routes.plex._shared import _content_libs, _norm_lang
|
||||
from app.routes.plex._shared import _norm_lang
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
@@ -43,7 +44,7 @@ SURPRISE_PER_KIND = 3 # Surprise-me: at most this many movies + this many shows
|
||||
|
||||
def _scope_libs(db: Session, scope: str) -> tuple[list[int], list[int]]:
|
||||
"""The movie/show library ids visible for this scope (empty list = out of scope)."""
|
||||
movie_ids, show_ids = _content_libs(db)
|
||||
movie_ids, show_ids = content_libs(db)
|
||||
return (
|
||||
movie_ids if scope in ("movie", "both") else [],
|
||||
show_ids if scope in ("show", "both") else [],
|
||||
@@ -112,10 +113,11 @@ def _recently_added(
|
||||
else []
|
||||
)
|
||||
# Merge the two type-homogeneous lists by added_at, then serialize each side in one batch so the
|
||||
# per-card state lookups stay batched (no N+1). The `(is not None, value)` key sorts a missing
|
||||
# timestamp last under reverse.
|
||||
# per-card state lookups stay batched (no N+1). Same `or NO_TS` key as the two rails below — the
|
||||
# E24 review found this one still on the older `(is not None, value)` form, i.e. one rule in two
|
||||
# spellings, with the two comments explaining it differently.
|
||||
tagged = [(m.added_at, "movie", m) for m in movies] + [(s.added_at, "show", s) for s in shows]
|
||||
tagged.sort(key=lambda t: (t[0] is not None, t[0]), reverse=True)
|
||||
tagged.sort(key=lambda t: t[0] or rollup.NO_TS, reverse=True)
|
||||
top = tagged[:RAIL_LIMIT]
|
||||
# Key the serialized cards back by rating_key (globally unique in Plex — a movie and a show never
|
||||
# share one), so the merge doesn't depend on object identity staying alive.
|
||||
@@ -170,7 +172,7 @@ def _recently_watched(
|
||||
)
|
||||
for ep, st, sh_title, sh_rk in db.execute(eq).all():
|
||||
rows.append((st.watched_at, leaf_card(ep, st, show_title=sh_title, show_rk=sh_rk)))
|
||||
rows.sort(key=lambda r: (r[0] is not None, r[0]), reverse=True)
|
||||
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
||||
return [card for _, card in rows[:RAIL_LIMIT]]
|
||||
|
||||
|
||||
@@ -234,7 +236,7 @@ def _continue_watching(
|
||||
for sh, resumed_at in srows:
|
||||
rows.append((resumed_at, show_card(sh, "in_progress")))
|
||||
|
||||
rows.sort(key=lambda r: (r[0] is not None, r[0]), reverse=True)
|
||||
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
||||
return [card for _, card in rows[:RAIL_LIMIT]]
|
||||
|
||||
|
||||
|
||||
@@ -19,8 +19,7 @@ from app.models import (
|
||||
from app.plex import rollup
|
||||
from app.plex import tmdb as plex_tmdb
|
||||
from app.plex.cards import movie_card, show_card
|
||||
from app.plex.query import split_csv
|
||||
from app.routes.plex._shared import _content_libs
|
||||
from app.plex.query import content_libs, split_csv
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
@@ -71,7 +70,7 @@ def people_cards(
|
||||
wanted = split_csv(names)[:16]
|
||||
if not wanted:
|
||||
return {"people": []}
|
||||
movie_lib_ids, show_lib_ids = _content_libs(db)
|
||||
movie_lib_ids, show_lib_ids = content_libs(db)
|
||||
want_movies = scope in ("movie", "both") and bool(movie_lib_ids)
|
||||
want_shows = scope in ("show", "both") and bool(show_lib_ids)
|
||||
photos = {
|
||||
@@ -185,7 +184,7 @@ def person_detail(
|
||||
and _should_schedule_enrich(person.id)
|
||||
):
|
||||
background_tasks.add_task(_enrich_person_bg, person.id)
|
||||
movie_lib_ids, show_lib_ids = _content_libs(db)
|
||||
movie_lib_ids, show_lib_ids = content_libs(db)
|
||||
|
||||
movies: list[dict] = []
|
||||
if movie_lib_ids:
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
"""Plex playback: image/subtitle proxies, HLS streaming, and watch-state writes."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
@@ -12,17 +10,20 @@ from fastapi import (
|
||||
BackgroundTasks,
|
||||
Depends,
|
||||
HTTPException,
|
||||
Path,
|
||||
Query,
|
||||
Request,
|
||||
Response,
|
||||
)
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user, resolved_user_id
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import (
|
||||
POSITION_MAX_S,
|
||||
PlexItem,
|
||||
PlexSeason,
|
||||
PlexShow,
|
||||
@@ -89,62 +90,70 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response
|
||||
return imgcache.response(data, ct)
|
||||
|
||||
|
||||
# Both subtitle routes authenticate with the full `current_user`, not the cookie-only
|
||||
# `_require_session` the image proxies use. That lighter gate exists for bursts of low-sensitivity
|
||||
# poster fetches and skips the is_active / is_suspended / session-epoch checks — acceptable for a
|
||||
# poster, not for a route that starts reads of the media library: a suspended or "log out
|
||||
# everywhere"-revoked cookie kept scheduling them. They are not bursty (one poll a second, one fetch
|
||||
# per track), so the DB user-load costs nothing that matters.
|
||||
#
|
||||
# Both path parameters are bounded, the same way on both routes: they become a cache FILE name
|
||||
# (`sub_<rating_key>_<ord>`), and an absurd one — a 250-digit key or ordinal — outgrew the OS's name
|
||||
# limit and turned the status poll into a 500. A Plex rating_key is a short number; the bound only
|
||||
# has to be generous.
|
||||
_RatingKey = Path(max_length=32)
|
||||
_SubOrd = Path(ge=0, le=9999)
|
||||
|
||||
|
||||
@router.get("/subtitle/{rating_key}/{sub_ord}/status")
|
||||
def subtitle_status(
|
||||
rating_key: str = _RatingKey,
|
||||
sub_ord: int = _SubOrd,
|
||||
user: User = Depends(current_user),
|
||||
# Bounded both ways: a non-finite position used to reach the scheduler's arithmetic and 500, and
|
||||
# an absurd one reached ffmpeg's `-ss`, which refused it.
|
||||
at: float = Query(0.0, ge=0.0, le=subtitles.MAX_POSITION_S, allow_inf_nan=False),
|
||||
) -> dict:
|
||||
"""Is this subtitle track showable yet — and if not, get it moving. Extraction runs in the
|
||||
background (see plex/subtitles.py for the measurements that forced it there), so the player polls
|
||||
this while it waits instead of hanging on a <track> that will not load for a minute or more.
|
||||
|
||||
`at` is the viewer's absolute position: the extractor windows around it, so what gets read first
|
||||
is the part actually about to be watched."""
|
||||
return subtitles.ensure(rating_key, sub_ord, at)
|
||||
|
||||
|
||||
@router.get("/subtitle/{rating_key}/{sub_ord}")
|
||||
def subtitle(rating_key: str, sub_ord: int, request: Request, offset: float = 0.0) -> Response:
|
||||
def subtitle(
|
||||
rating_key: str = _RatingKey,
|
||||
sub_ord: int = _SubOrd,
|
||||
user: User = Depends(current_user),
|
||||
# Finite, like every float this router takes: `-inf` crashed the shift (a 500), `nan` stacked
|
||||
# every cue at 0. It can legitimately be negative (a subtitle nudged later than the session start).
|
||||
offset: float = Query(0.0, allow_inf_nan=False),
|
||||
) -> Response:
|
||||
"""Serve the ord-th subtitle track as WebVTT for the browser's <track> — so choosing a subtitle
|
||||
overlays it directly (no HLS re-mux / session restart). Handles EXTERNAL sidecar subs (fetched
|
||||
from Plex via the stream `key` — the old `-map 0:s` broke on these since they're not in the file)
|
||||
AND embedded text subs (extracted from the local file). Image subs (PGS/VobSub) → 415. Disk-cached."""
|
||||
_require_session(request)
|
||||
# Cache the ABSOLUTE-time VTT once (per rk+ord); the per-session cue-shift is applied on the way
|
||||
# out, so a resume/seek to any offset reuses the same cached track (no cache-per-offset explosion).
|
||||
cache_key = f"sub_{rating_key}_{sub_ord}"
|
||||
hit = subtitles.cache_read(cache_key)
|
||||
if hit is not None:
|
||||
return Response(subtitles.shift_vtt(hit, offset), media_type="text/vtt")
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
it = item_or_404(db, rating_key)
|
||||
with PlexClient(db) as plex:
|
||||
part = ((plex.metadata(it.rating_key) or {}).get("Media") or [{}])[0].get(
|
||||
"Part"
|
||||
) or [{}]
|
||||
subs = [s for s in (part[0].get("Stream") or []) if s.get("streamType") == 3]
|
||||
if sub_ord < 0 or sub_ord >= len(subs):
|
||||
raise HTTPException(status_code=404, detail="No such subtitle")
|
||||
s = subs[sub_ord]
|
||||
codec = (s.get("codec") or s.get("format") or "").lower()
|
||||
if codec in subtitles.BITMAP_SUBS:
|
||||
raise HTTPException(
|
||||
status_code=415,
|
||||
detail="Image-based subtitle can't be shown as text",
|
||||
)
|
||||
if s.get("key"): # external sidecar → fetch from Plex, convert to VTT
|
||||
raw = plex.raw_get(s["key"])
|
||||
if codec in ("srt", "subrip", ""):
|
||||
vtt = subtitles.srt_to_vtt(raw.decode("utf-8", "replace"))
|
||||
else:
|
||||
fd, tmp = tempfile.mkstemp(suffix="." + (codec or "sub"))
|
||||
try:
|
||||
os.write(fd, raw)
|
||||
os.close(fd)
|
||||
vtt = subtitles.ffmpeg_to_vtt(tmp, None)
|
||||
finally:
|
||||
os.unlink(tmp)
|
||||
elif s.get("index") is not None: # embedded → extract from the local file
|
||||
src = plex_paths.local_media_path(db, it.file_path)
|
||||
if src is None:
|
||||
raise HTTPException(status_code=404, detail="Media file not found on disk")
|
||||
emb_ord = sum(1 for x in subs[:sub_ord] if x.get("index") is not None)
|
||||
vtt = subtitles.ffmpeg_to_vtt(str(src), emb_ord)
|
||||
else:
|
||||
raise HTTPException(status_code=404, detail="Subtitle not fetchable")
|
||||
except PlexNotConfigured as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except PlexError as e:
|
||||
raise HTTPException(status_code=502, detail=f"Subtitle fetch failed: {e}")
|
||||
subtitles.cache_write(cache_key, vtt)
|
||||
return Response(subtitles.shift_vtt(vtt, offset), media_type="text/vtt")
|
||||
AND embedded text subs (extracted from the local file). Image subs (PGS/VobSub) → 415. Disk-cached.
|
||||
|
||||
A pure READ: it serves whatever is cached and schedules nothing — the status poll owns that,
|
||||
because only it knows where the viewer is (see `subtitles.cached_track`). The player mounts the
|
||||
<track> only once /status says there is something to show, so in practice this is only ever hit
|
||||
warm; on a cache eviction race the 202 sends the player back to polling."""
|
||||
# The cached VTT holds ABSOLUTE cue times (per rk+ord); the per-session cue-shift is applied on
|
||||
# the way out, so a resume/seek to any offset reuses the same track (no cache-per-offset explosion).
|
||||
hit, code = subtitles.cached_track(rating_key, sub_ord)
|
||||
if hit is None and code:
|
||||
raise HTTPException(
|
||||
status_code=subtitles.UNAVAILABLE_STATUS.get(code, 422),
|
||||
detail={"code": "subtitle", "reason": code},
|
||||
)
|
||||
if hit is None:
|
||||
return Response(
|
||||
status_code=202, media_type="application/json", content='{"state":"preparing"}'
|
||||
)
|
||||
return Response(subtitles.shift_vtt(hit, offset), media_type="text/vtt")
|
||||
|
||||
|
||||
# Resume/watched thresholds. _PROGRESS_MIN_S matches the feed's PROGRESS_MIN_SECONDS (5), but
|
||||
@@ -287,10 +296,15 @@ def _push_watch(
|
||||
|
||||
|
||||
class ItemProgressIn(BaseModel):
|
||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||
# one into a 500.
|
||||
model_config = ConfigDict(allow_inf_nan=False)
|
||||
|
||||
# float, truncated in the handler — the player may post a raw currentTime (a float); a plain
|
||||
# `int` field would 422 it on the periodic progress checkpoint (the old `int(...)` truncated).
|
||||
position_seconds: float = 0
|
||||
duration_seconds: float = 0
|
||||
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
||||
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
final: bool = False
|
||||
|
||||
|
||||
@@ -309,6 +323,14 @@ def item_progress(
|
||||
`_should_push_resume`). `final` (sent on pause / seek / pagehide / unmount) pushes immediately and
|
||||
resets the throttle window, so Plex sees a settled position the instant the user pauses or seeks."""
|
||||
it = item_or_404(db, rating_key)
|
||||
# A player parked mid-item — playing or PAUSED — checkpoints every 10 s, so this is the signal
|
||||
# that such a viewer still wants their HLS session: the one thing `last_access` could not see,
|
||||
# since it moved only on a file fetch and a paused player fetches nothing (see
|
||||
# stream.reap_idle). It is NOT every open player: `usePlexProgress.flush` sends nothing once the
|
||||
# finish latch is up, or before the first position is known. Those are the states in which
|
||||
# reaping is the right answer anyway, and a viewer who scrubs back into a reaped session gets
|
||||
# the same 404 → reclaim the player already handles.
|
||||
plex_stream.touch_session(str(rating_key))
|
||||
position = max(0, int(body.position_seconds))
|
||||
duration = int(body.duration_seconds)
|
||||
final = body.final
|
||||
@@ -406,10 +428,20 @@ def item_state(
|
||||
)
|
||||
prev_status = st.status if st is not None else "new"
|
||||
if status == "new":
|
||||
# Does Plex hold anything for this item that the un-mark has to clear? Two things qualify: a
|
||||
# view count, and a resume point. An IN-PROGRESS item carries status "new" WITH a position —
|
||||
# that is how the importer maps a bare `viewOffset` (`_plex_watch_to_state`) — so keying the
|
||||
# push on the status alone silently skipped the single case "Clear resume point" exists for:
|
||||
# the un-mark never reached Plex, the offset stayed, and the next pull imported it straight
|
||||
# back (Protector, cleared twice by the user and back at 92s both times). `hidden` is
|
||||
# Siftlode-only and still pushes nothing, exactly as this endpoint's docstring says.
|
||||
mirrored_on_plex = st is not None and (
|
||||
st.status == "watched" or (st.position_seconds or 0) > 0
|
||||
)
|
||||
if st is not None:
|
||||
db.delete(st)
|
||||
db.commit()
|
||||
if prev_status != "new": # something → unwatched: mirror the un-mark to Plex
|
||||
if mirrored_on_plex:
|
||||
_push_watch(background, db, user.id, it.id, rating_key, "unwatched")
|
||||
return {"status": "new"}
|
||||
if st is None:
|
||||
@@ -524,9 +556,10 @@ def season_state(
|
||||
@router.post("/stream/{rating_key}/session")
|
||||
def stream_session(
|
||||
rating_key: str,
|
||||
start: float = 0.0,
|
||||
# Finite: `start=inf` reached `int()` in the session builder and came back as a 500.
|
||||
start: float = Query(0.0, allow_inf_nan=False),
|
||||
audio: int | None = None,
|
||||
aoff: float = 0.0,
|
||||
aoff: float = Query(0.0, allow_inf_nan=False),
|
||||
multi: bool = False,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
@@ -536,7 +569,6 @@ def stream_session(
|
||||
HEVC/VP9/AV1/…) take the same HLS path but with the video re-encoded to H.264 (VAAPI or libx264).
|
||||
Selecting a non-default audio track (`audio` = stream ordinal) forces the HLS path so the chosen audio
|
||||
can be muxed in. Subtitles are independent WebVTT tracks (GET /subtitle) — no session restart."""
|
||||
plex_stream.reap_idle() # opportunistic cleanup of idle sessions on each new playback
|
||||
it = item_or_404(db, rating_key)
|
||||
transcode = it.playable == "transcode"
|
||||
# `multi` (item has ≥2 audio tracks) forces the HLS path so ALL audio tracks ship as renditions and
|
||||
@@ -560,7 +592,9 @@ def stream_session(
|
||||
return {
|
||||
"mode": "hls",
|
||||
"start_s": s.media_start_s,
|
||||
"url": f"/api/plex/stream/{it.rating_key}/hls/{s.entry}",
|
||||
# The session id is part of the path (not a query param) so the playlist's RELATIVE segment
|
||||
# names resolve back into the same session — see `stream_hls_file` for why it has to.
|
||||
"url": f"/api/plex/stream/{it.rating_key}/hls/{s.id}/{s.entry}",
|
||||
"transcode": s.transcode,
|
||||
"hw": s.hw,
|
||||
"tonemap": s.tonemap,
|
||||
@@ -592,45 +626,82 @@ def stream_file(
|
||||
return FileResponse(str(src)) # Starlette honours the Range header (206)
|
||||
|
||||
|
||||
# What a session directory can contain, and nothing else. No `.vtt`: ffmpeg runs with `-sn` and
|
||||
# subtitles are independent tracks served by GET /subtitle, so a WebVTT name here could only ever be
|
||||
# a name that will not arrive — and accepting it cost a 20 s parked worker per request.
|
||||
_HLS_CT = {
|
||||
".m3u8": "application/vnd.apple.mpegurl",
|
||||
".ts": "video/mp2t",
|
||||
".vtt": "text/vtt",
|
||||
}
|
||||
# A real artifact name is ~20 characters. The bound is what keeps an absurd one out of the parser:
|
||||
# `int()` on an unbounded digit run raises ValueError past 4300 digits (CPython's
|
||||
# `sys.get_int_max_str_digits`), which reached the client as a 500 — the same shape of bug the
|
||||
# subtitle routes were bounded for (`_RatingKey`, above).
|
||||
_HLS_NAME_MAX = 128
|
||||
|
||||
|
||||
@router.get("/stream/{rating_key}/hls/{filename}")
|
||||
@router.get("/stream/{rating_key}/hls/{session_id}/{filename}")
|
||||
def stream_hls_file(
|
||||
rating_key: str,
|
||||
session_id: str,
|
||||
filename: str,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> Response:
|
||||
"""Serve a file (playlist / segment / subtitle) from the current HLS session directory. The
|
||||
playlist references its segments/subtitles by relative name, all resolving back here."""
|
||||
"""Serve a file (playlist / segment / subtitle) from ONE HLS session's directory. The playlist
|
||||
references its segments/subtitles by relative name, all resolving back here.
|
||||
|
||||
The session id in the path is what makes "back here" mean the same session. Sessions are keyed by
|
||||
rating_key alone and a seek-restart or a second viewer of the same item replaces the registered
|
||||
one, while the DIRECTORY is per (key, offset, tag) — so resolving by rating_key handed the old
|
||||
viewer the new session's `seg_12.ts`, produced from a different offset under `-copyts`: wrong
|
||||
bytes with a 200, which the player has no way to notice (deep-review round 5). A request for a
|
||||
session that is no longer the registered one is a 404, and the player reclaims at the playhead.
|
||||
|
||||
That reclaim is also why a request with no live session is no longer auto-started here: the
|
||||
auto-start could only ever begin at 0, silently throwing away the viewer's position."""
|
||||
# Filename must be a plain hls artifact — no path separators, only the expected extensions.
|
||||
ext = "." + filename.rsplit(".", 1)[-1] if "." in filename else ""
|
||||
if ext not in _HLS_CT or "/" in filename or "\\" in filename or ".." in filename:
|
||||
if (
|
||||
len(filename) > _HLS_NAME_MAX
|
||||
or ext not in _HLS_CT
|
||||
or "/" in filename
|
||||
or "\\" in filename
|
||||
or ".." in filename
|
||||
):
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
s = plex_stream.current_session(str(rating_key))
|
||||
if s is None:
|
||||
# No session yet (e.g. reaped): the ENTRY playlist may auto-start one at 0 — master.m3u8 for a
|
||||
# multi-audio item, index.m3u8 for a single-audio remux item.
|
||||
it = item_or_404(db, rating_key)
|
||||
if filename == "master.m3u8":
|
||||
s = plex_stream.start_session(db, it, 0.0, multi=True)
|
||||
elif filename == "index.m3u8" and it.playable == "remux":
|
||||
s = plex_stream.start_session(db, it, 0.0)
|
||||
else:
|
||||
raise HTTPException(status_code=404, detail="No active playback session")
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Media file not found on disk")
|
||||
if s is None or s.id != session_id:
|
||||
raise HTTPException(status_code=404, detail="No active playback session")
|
||||
path = s.dir / filename
|
||||
timeout = 30 if ext == ".m3u8" else 20
|
||||
if not plex_stream.wait_for(path, timeout):
|
||||
# Only a name THIS session can actually produce is worth waiting for: its own playlists, and a
|
||||
# segment of one of its own renditions whose predecessor is already on disk (ffmpeg writes them
|
||||
# in order, so if it is not there this one is not next). The session is passed, not just its
|
||||
# directory, because the single- and multi-audio argv branches spell their names differently and
|
||||
# neither can produce the other's. Answering now instead of waiting it out is what keeps a seek
|
||||
# past the produced region — or a deliberate flood of hopeless names — from parking threadpool
|
||||
# workers and their DB connections (see `artifact_may_arrive`).
|
||||
if not plex_stream.artifact_may_arrive(s, filename):
|
||||
raise HTTPException(status_code=404, detail="Not available yet")
|
||||
timeout = 30 if ext == ".m3u8" else 20
|
||||
# `proc` so a session whose ffmpeg has already exited answers at once instead of polling out the
|
||||
# whole timeout: this is a SYNC route on the ~40-slot threadpool, holding a pooled DB connection
|
||||
# with it, so every such wait is a worker (and a connection) parked for 20 s — an ordinary seek
|
||||
# past the produced region was enough to do it (round 5).
|
||||
if not plex_stream.wait_for(path, timeout, proc=s.proc):
|
||||
raise HTTPException(status_code=404, detail="Not available yet")
|
||||
try:
|
||||
content = path.read_bytes()
|
||||
except OSError:
|
||||
# The session was torn down between the wait and the read — `_kill` rmtrees the whole
|
||||
# directory, and nothing holds `_lock` across those two steps. It has to present as the same
|
||||
# 404 a gone session does: the player's recovery is keyed on the STATUS (`code === 404`), so
|
||||
# a 500 here skipped the reclaim and ended playback with "couldn't play" instead of
|
||||
# restarting at the playhead. Ordinary trigger: `loadSession` awaits the new session — which
|
||||
# kills and rmtrees the old one — before destroying the old hls instance, so the outgoing
|
||||
# instance's in-flight segment request races the rmtree on every seek-restart.
|
||||
raise HTTPException(status_code=404, detail="No active playback session") from None
|
||||
return Response(
|
||||
content=path.read_bytes(),
|
||||
content=content,
|
||||
media_type=_HLS_CT[ext],
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
@@ -25,7 +25,7 @@ router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
|
||||
def _own_playlist_or_404(db: Session, pid: int, user: User) -> PlexPlaylist:
|
||||
return owned_or_404(db, PlexPlaylist, pid, user, detail="Playlist not found")
|
||||
return owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
|
||||
|
||||
def _playlist_card(pl: PlexPlaylist, count: int, thumb_rk: str | None) -> dict:
|
||||
|
||||
@@ -6,7 +6,7 @@ the app or any account data. Password-protected links exchange the password once
|
||||
a short-lived signed grant that the media URL carries (a `<video src>` can't send a header).
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
@@ -17,13 +17,21 @@ from app.db import get_db
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, DownloadLink, MediaAsset
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import verify_password
|
||||
|
||||
router = APIRouter(prefix="/api/public", tags=["public"])
|
||||
|
||||
# Slow password guessing per link (the token itself is 192-bit, so it needs no throttle).
|
||||
# Slow password guessing on a link (the token itself is 192-bit, so it needs no throttle). Counted per
|
||||
# viewer — the link AND the client (`client_key`) — so a client guessing wrong shuts out only itself, and a
|
||||
# busy link's own unlocks don't count against its next viewer.
|
||||
_unlock_limit = RateLimiter(max_events=10, window_seconds=300)
|
||||
# ...under a ceiling on WRONG passwords for the link from every client together, so guessing spread over
|
||||
# many addresses stays slow too; unlocking with the password never adds to it. The price of the ceiling:
|
||||
# someone with the link but not its password, guessing from 50 / 10 = 5 networks, can keep the link shut
|
||||
# for everyone until the window passes. It is checked before the password and counted after it, so guesses
|
||||
# already running when it fills are still checked — as many as the route's threadpool runs at once.
|
||||
_unlock_failures = RateLimiter(max_events=50, window_seconds=300)
|
||||
|
||||
|
||||
def _link_or_404(db: Session, token: str) -> DownloadLink:
|
||||
@@ -83,11 +91,16 @@ class WatchUnlockIn(BaseModel):
|
||||
|
||||
|
||||
@router.post("/watch/{token}/unlock")
|
||||
def watch_unlock(token: str, body: WatchUnlockIn, db: Session = Depends(get_db)) -> dict:
|
||||
def watch_unlock(
|
||||
token: str, body: WatchUnlockIn, request: Request, db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Exchange the link password for a signed grant carried by the media URL."""
|
||||
if not _unlock_limit.allow(token):
|
||||
raise HTTPException(status_code=429, detail="Too many attempts. Try again later.")
|
||||
# The link first: a token naming none must leave no limiter entry behind, or posting made-up tokens
|
||||
# would grow the limiter without bound.
|
||||
link = _link_or_404(db, token)
|
||||
viewer = f"{token}|{client_key(request)}"
|
||||
if _unlock_failures.exhausted(token) or not _unlock_limit.allow(viewer):
|
||||
raise HTTPException(status_code=429, detail="Too many attempts. Try again later.")
|
||||
if not link.password_hash:
|
||||
# Not a password link — nothing to unlock; just hand back the plain meta.
|
||||
job, asset = _ready_target(db, link)
|
||||
@@ -96,6 +109,7 @@ def watch_unlock(token: str, body: WatchUnlockIn, db: Session = Depends(get_db))
|
||||
**linksmod.public_meta(link, job, asset, _file_url(token), _poster_url(token, asset)),
|
||||
}
|
||||
if not verify_password(body.password, link.password_hash):
|
||||
_unlock_failures.allow(token)
|
||||
raise HTTPException(status_code=403, detail="Wrong password.")
|
||||
job, asset = _ready_target(db, link)
|
||||
link.view_count += 1
|
||||
|
||||
@@ -91,7 +91,7 @@ def create_view(
|
||||
|
||||
|
||||
def _own_view(db: Session, user: User, view_id: int) -> SavedView:
|
||||
return owned_or_404(db, SavedView, view_id, user, detail="Unknown view")
|
||||
return owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
|
||||
|
||||
class UpdateViewIn(BaseModel):
|
||||
|
||||
@@ -22,7 +22,9 @@ from app.scheduler import (
|
||||
trigger_all,
|
||||
trigger_job,
|
||||
)
|
||||
from app.sync.explore import followed_channel
|
||||
from app.sync.runner import estimate_deep_backfill
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/admin/scheduler", tags=["admin"])
|
||||
|
||||
@@ -41,7 +43,7 @@ def set_job_interval(
|
||||
"""Change how often a scheduler job runs (minutes). Persisted as an override and applied
|
||||
to the live scheduler immediately."""
|
||||
if job_id not in JOB_INTERVALS:
|
||||
raise HTTPException(status_code=404, detail="Unknown job")
|
||||
raise not_found("job")
|
||||
minutes = body.interval_minutes
|
||||
if not (MIN_INTERVAL <= minutes <= MAX_INTERVAL):
|
||||
raise HTTPException(
|
||||
@@ -73,7 +75,7 @@ def run_job_now(
|
||||
try:
|
||||
result = trigger_job(job_id, actor_id=user.id)
|
||||
except KeyError:
|
||||
raise HTTPException(status_code=404, detail="Unknown job")
|
||||
raise not_found("job")
|
||||
if result == "already_running":
|
||||
raise HTTPException(status_code=409, detail="Job is already running")
|
||||
return {"id": job_id, "started": True}
|
||||
@@ -136,9 +138,17 @@ def get_scheduler(_: User = Depends(admin_user), db: Session = Depends(get_db))
|
||||
|
||||
queue = {
|
||||
"channels_recent_pending": db.scalar(
|
||||
select(func.count()).select_from(Channel).where(Channel.recent_synced_at.is_(None))
|
||||
# The same selection run_recent_backfill works through, or the card counts channels
|
||||
# the job will never touch and never reaches zero.
|
||||
select(func.count())
|
||||
.select_from(Channel)
|
||||
.where(Channel.recent_synced_at.is_(None), followed_channel())
|
||||
)
|
||||
or 0,
|
||||
# The whole opted-in BACKLOG, deliberately not `run_deep_backfill`'s next-pass selection
|
||||
# (which also wants `recent_synced_at`): the same figure /api/sync/my-status shows the user,
|
||||
# and a channel missing its recent pull is one the same job tick fetches first, not one the
|
||||
# job will never touch. Narrowing it here would under-report work that is genuinely queued.
|
||||
"channels_deep_pending": deep_eta["channels_pending"],
|
||||
"deep_eta_seconds": deep_eta["eta_seconds"],
|
||||
"videos_pending_enrich": db.scalar(
|
||||
|
||||
@@ -231,9 +231,7 @@ def search_youtube(
|
||||
limit = max(1, min(limit, 100))
|
||||
|
||||
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
|
||||
source = (sysconfig.get_str(db, "search_source") or "scrape").lower()
|
||||
if source != "scrape":
|
||||
source = "api"
|
||||
source = sysconfig.get_str(db, "search_source")
|
||||
|
||||
# Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
|
||||
# since the search itself spends no quota.
|
||||
|
||||
@@ -80,7 +80,7 @@ def create_tag(
|
||||
|
||||
def _own_tag(db: Session, user: User, tag_id: int) -> Tag:
|
||||
# System tags (user_id NULL) are not user-editable → owned_or_404 404s them too.
|
||||
return owned_or_404(db, Tag, tag_id, user, detail="Unknown tag")
|
||||
return owned_or_404(db, Tag, tag_id, user, what="tag")
|
||||
|
||||
|
||||
class TagUpdateIn(BaseModel):
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Callable
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import audit, progress, quota
|
||||
from app import audit, logsink, progress, quota
|
||||
from app.audit import AuditAction
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
@@ -72,6 +72,7 @@ JOB_INTERVALS: dict[str, int] = {
|
||||
"plex_watch_reconcile": settings.plex_watch_reconcile_interval_min,
|
||||
"tmdb_enrich": settings.tmdb_enrich_interval_min,
|
||||
"audit_gc": settings.audit_gc_minutes,
|
||||
"log_gc": settings.log_gc_minutes,
|
||||
}
|
||||
|
||||
# Sane bounds for an admin-set interval (minutes).
|
||||
@@ -381,6 +382,19 @@ def _audit_gc_job() -> None:
|
||||
_job("audit_gc", work)
|
||||
|
||||
|
||||
def _log_gc_job() -> None:
|
||||
# Prune log_events rows older than log_retention_days — the limit the privacy policy states, so
|
||||
# this runs hourly rather than daily. Pure DB, no quota.
|
||||
def work(db):
|
||||
from app import sysconfig
|
||||
|
||||
reaped = logsink.gc(db, sysconfig.get_int(db, "log_retention_days"))
|
||||
db.commit()
|
||||
return {"reaped": reaped}
|
||||
|
||||
_job("log_gc", work)
|
||||
|
||||
|
||||
# job_id -> wrapper. The single source of truth for which jobs exist and how to run one,
|
||||
# shared by start_scheduler (recurring registration) and trigger_job (manual "run now").
|
||||
JOB_FUNCS: dict[str, Callable[[], None]] = {
|
||||
@@ -400,6 +414,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = {
|
||||
"plex_watch_reconcile": _plex_watch_reconcile_job,
|
||||
"tmdb_enrich": _tmdb_enrich_job,
|
||||
"audit_gc": _audit_gc_job,
|
||||
"log_gc": _log_gc_job,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ grace period (mirrors the maintenance job's grace-then-delete safety)."""
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
@@ -31,6 +31,23 @@ from app.youtube.client import YouTubeClient
|
||||
log = logging.getLogger("siftlode.explore")
|
||||
|
||||
|
||||
def followed_channel():
|
||||
"""WHERE clause: a channel somebody follows — a subscription, or an exploration still inside its
|
||||
grace window (`purge_unkept_explored` deletes the expired ExploredChannel rows first). ONE
|
||||
definition for both halves of the question: what the scheduled RSS poll and recent backfill cover
|
||||
(sync/runner.py) and what the explore purge may delete.
|
||||
|
||||
The poll and backfill cover only these (owner decision, 2026-09-17). They used to cover EVERY
|
||||
channel row, but rows also come from search results, playlist sync and dropped subscriptions, and
|
||||
nothing deletes those: on prod 582 of 979 polled channels had no follower, three of them dead
|
||||
feeds answering 404 on every poll. The cost of narrowing is deliberate: the shared Library stops
|
||||
receiving new uploads from channels nobody follows."""
|
||||
return or_(
|
||||
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists(),
|
||||
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists(),
|
||||
)
|
||||
|
||||
|
||||
def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
|
||||
"""Enrich just the not-yet-enriched videos among `video_ids` (videos.list, 1 unit/50)."""
|
||||
if not video_ids:
|
||||
@@ -98,12 +115,8 @@ def purge_unkept_explored(db: Session) -> dict:
|
||||
).rowcount
|
||||
db.commit()
|
||||
|
||||
sub_exists = select(Subscription.id).where(Subscription.channel_id == Channel.id).exists()
|
||||
exp_exists = select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists()
|
||||
orphan_ids = (
|
||||
db.execute(
|
||||
select(Channel.id).where(Channel.from_explore.is_(True), ~sub_exists, ~exp_exists)
|
||||
)
|
||||
db.execute(select(Channel.id).where(Channel.from_explore.is_(True), ~followed_channel()))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
+59
-17
@@ -7,16 +7,18 @@ public reads.
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, quota, sysconfig
|
||||
from app import logs, progress, quota, sysconfig
|
||||
|
||||
log = logging.getLogger("siftlode.sync")
|
||||
from app.db import SessionLocal
|
||||
from app.models import Channel, OAuthToken, Subscription, User
|
||||
from app.sync.explore import followed_channel
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
from app.sync.videos import (
|
||||
apply_rss_feed,
|
||||
@@ -32,6 +34,8 @@ from app.youtube.rss import RssError, fetch_channel_feed
|
||||
|
||||
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
|
||||
RSS_POLL_WORKERS = 16
|
||||
# Failed channels named in a poll's summary warning; the rest are counted by reason.
|
||||
RSS_FAILURE_EXAMPLES = 5
|
||||
|
||||
|
||||
def token_users(db: Session) -> list[User]:
|
||||
@@ -52,7 +56,7 @@ def get_service_user(db: Session) -> User | None:
|
||||
|
||||
|
||||
def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str, int]:
|
||||
"""Poll every channel's free RSS feed. Returns `{"new": inserted, "failed": channels_that_did
|
||||
"""Poll the free RSS feed of every followed channel (`followed_channel`). Returns `{"new": inserted, "failed": channels_that_did
|
||||
_not_complete}` — the failure count is part of the RESULT, not just a log line, so the scheduler
|
||||
card and the audit summary can tell "no new uploads" apart from "nothing worked" (an egress
|
||||
outage, or a database that can't accept writes, otherwise renders identically to a quiet hour).
|
||||
@@ -67,7 +71,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
carries the FIRST underlying error, because on a small instance "all channels failed" is also
|
||||
what a single dead feed id looks like."""
|
||||
if channels is None:
|
||||
channels = db.execute(select(Channel)).scalars().all()
|
||||
channels = db.execute(select(Channel).where(followed_channel())).scalars().all()
|
||||
total = len(channels)
|
||||
new = 0
|
||||
done = 0
|
||||
@@ -76,6 +80,27 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
# outage from one permanently dead feed id — which is the whole difference on an instance with
|
||||
# a single subscription, where BOTH render as 100% failed.
|
||||
first_error: str | None = None
|
||||
# What failed, as counts per reason (`http_404`, `transport`, `apply`, …) plus a few examples —
|
||||
# the ONE warning a poll writes. One line per failed channel was what turned YouTube's RSS outage
|
||||
# of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now.
|
||||
reasons: Counter[str] = Counter()
|
||||
examples: list[str] = []
|
||||
|
||||
def fail(reason: str, detail: str, *, unexpected: str | None = None) -> None:
|
||||
"""Count a channel that did not get polled. `unexpected` is the message for a failure that
|
||||
is a bug on our side: its FIRST occurrence per reason gets a traceback at ERROR, the rest
|
||||
DEBUG — a read-only database fails every channel the same way, and 979 identical tracebacks
|
||||
tell nobody anything the first one didn't. Call it from inside the `except` block."""
|
||||
nonlocal failed, first_error
|
||||
if unexpected is not None:
|
||||
level = logging.ERROR if reasons[reason] == 0 else logging.DEBUG
|
||||
log.log(level, unexpected, exc_info=True)
|
||||
failed += 1
|
||||
reasons[reason] += 1
|
||||
first_error = first_error or detail
|
||||
if len(examples) < RSS_FAILURE_EXAMPLES:
|
||||
examples.append(detail)
|
||||
|
||||
# Fetch feeds concurrently (network-bound, no DB), then apply inserts on this thread as
|
||||
# each fetch returns — a SQLAlchemy session isn't thread-safe. Pass the channel id (a
|
||||
# plain str) to the workers; the ORM instance is touched only here on the main thread.
|
||||
@@ -90,16 +115,15 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
# A failed poll is NOT an empty feed: skip the apply so `last_rss_at` stays where it
|
||||
# was (stamping it made a permanently-broken feed look freshly polled). The shared
|
||||
# counter/progress tail lives after the statement, so the two outcomes can stay two
|
||||
# clauses: an RssError is EXPECTED (404, timeout) and gets one warning line; anything
|
||||
# else is a bug on our side and earns a traceback.
|
||||
# clauses: an RssError is EXPECTED (404, timeout) and is only counted; anything else is a
|
||||
# bug on our side and earns a traceback (the first of its kind per poll).
|
||||
except RssError as exc:
|
||||
log.warning("RSS poll skipped for channel %s: %s", channel.id, exc)
|
||||
failed += 1
|
||||
first_error = first_error or f"{channel.id}: {exc}"
|
||||
log.debug("RSS poll skipped for channel %s: %s", channel.id, exc)
|
||||
fail(exc.reason, f"{channel.id}: {exc}")
|
||||
except Exception as exc:
|
||||
log.exception("RSS fetch failed for channel %s", channel.id)
|
||||
failed += 1
|
||||
first_error = first_error or f"{channel.id}: {exc}"
|
||||
fail(
|
||||
"error", f"{channel.id}: {exc}", unexpected=f"RSS fetch failed for {channel.id}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
new += apply_rss_feed(db, channel, entries)
|
||||
@@ -107,14 +131,27 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
# A write failure (read-only DB, failover) leaves the channel just as un-polled
|
||||
# as an unreachable feed — count it, or a wholly broken run reports "failed=0".
|
||||
db.rollback()
|
||||
failed += 1
|
||||
first_error = first_error or f"{channel.id} (apply): {exc}"
|
||||
log.exception("RSS apply failed for channel %s", channel.id)
|
||||
fail(
|
||||
"apply",
|
||||
f"{channel.id} (apply): {exc}",
|
||||
unexpected=f"RSS apply failed for {channel.id}",
|
||||
)
|
||||
# One place that advances the counter, whatever happened above.
|
||||
done += 1
|
||||
progress.report(done, total, "rss_poll")
|
||||
if failed:
|
||||
log.warning("RSS poll: %d/%d channels could not be polled", failed, total)
|
||||
log.warning(
|
||||
"RSS poll: %d/%d channels could not be polled",
|
||||
failed,
|
||||
total,
|
||||
extra=logs.event(
|
||||
"sync.rss.failed",
|
||||
failed=failed,
|
||||
total=total,
|
||||
reasons=dict(reasons),
|
||||
examples=examples,
|
||||
),
|
||||
)
|
||||
if total and failed == total:
|
||||
# Nothing was polled at all — an egress outage (the tinyproxy-after-reboot incident), a
|
||||
# dead DNS, a read-only database. Returning normally would record the run as `ok` with the
|
||||
@@ -174,7 +211,11 @@ def run_recent_backfill(
|
||||
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
|
||||
if channels is None:
|
||||
channels = (
|
||||
db.execute(select(Channel).where(Channel.recent_synced_at.is_(None))).scalars().all()
|
||||
db.execute(
|
||||
select(Channel).where(Channel.recent_synced_at.is_(None), followed_channel())
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if max_channels:
|
||||
channels = channels[:max_channels]
|
||||
@@ -204,7 +245,8 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
if user is None:
|
||||
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
|
||||
# Demand-driven: only deep-backfill channels at least one user has opted into
|
||||
# (Subscription.deep_requested). Recent backfill stays unconditional (cheap).
|
||||
# (Subscription.deep_requested). Recent backfill needs no opt-in: it covers every followed channel
|
||||
# (`followed_channel`).
|
||||
deep_requested = (
|
||||
select(Subscription.id)
|
||||
.where(
|
||||
|
||||
+50
-19
@@ -14,7 +14,7 @@ from dataclasses import dataclass
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import security
|
||||
from app.config import settings
|
||||
from app.config import Settings, canonical_choice, choices_of, settings
|
||||
from app.models import SystemConfig
|
||||
|
||||
log = logging.getLogger("siftlode.config")
|
||||
@@ -27,7 +27,7 @@ _decrypt_warned: set[str] = set()
|
||||
@dataclass(frozen=True)
|
||||
class ConfigSpec:
|
||||
key: str
|
||||
type: str # "int" | "str" | "bool"
|
||||
type: str # "int" | "str" | "bool" | "choice"
|
||||
group: str # admin-UI grouping (logically related keys stay together)
|
||||
default_attr: str # attribute on `settings` holding the env/config default
|
||||
secret: bool = False
|
||||
@@ -35,16 +35,33 @@ class ConfigSpec:
|
||||
max: int | None = None
|
||||
allow_empty: bool = False # str keys: a stored "" is an explicit empty value, not "unset"
|
||||
|
||||
@property
|
||||
def choices(self) -> tuple[str, ...]:
|
||||
"""A choice key's allowed values, read from its setting's Literal type in config.py — the one
|
||||
place the list is written. The admin page renders them as a picker."""
|
||||
return choices_of(Settings.model_fields[self.default_attr].annotation)
|
||||
|
||||
|
||||
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
|
||||
# off env into the admin UI; the admin page renders it automatically from this list.
|
||||
# off env into the admin UI; the admin page renders it automatically from this list. The ORDER is the
|
||||
# admin page's tab order (first appearance of each group), so keep a group's keys together.
|
||||
SPECS: tuple[ConfigSpec, ...] = (
|
||||
# --- Access / registration ---
|
||||
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
|
||||
# --- Email / SMTP (one logical group; the password is the only secret) ---
|
||||
ConfigSpec("smtp_host", "str", "email", "smtp_host"),
|
||||
ConfigSpec("smtp_port", "int", "email", "smtp_port", min=1, max=65535),
|
||||
ConfigSpec("smtp_user", "str", "email", "smtp_user", allow_empty=True),
|
||||
ConfigSpec("smtp_from", "str", "email", "smtp_from"),
|
||||
ConfigSpec("smtp_password", "str", "email", "smtp_password", secret=True),
|
||||
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
|
||||
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True),
|
||||
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).
|
||||
ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True),
|
||||
# --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the
|
||||
# install wizard / Configuration page so no restart is needed when the creds change. ---
|
||||
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True),
|
||||
ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True),
|
||||
# --- Quota (shared YouTube Data API daily budget) ---
|
||||
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
|
||||
ConfigSpec(
|
||||
@@ -59,7 +76,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
max=100_000,
|
||||
),
|
||||
# Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
|
||||
ConfigSpec("search_source", "str", "quota", "search_source"),
|
||||
ConfigSpec("search_source", "choice", "quota", "search_source"),
|
||||
# --- Backfill (recent-first first pass per channel) ---
|
||||
ConfigSpec(
|
||||
"backfill_recent_max_videos",
|
||||
@@ -96,16 +113,6 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650),
|
||||
# Days an un-kept live-search result video is kept before the cleanup job removes it.
|
||||
ConfigSpec("search_grace_days", "int", "explore", "search_grace_days", min=0, max=3_650),
|
||||
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
|
||||
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True),
|
||||
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).
|
||||
ConfigSpec("youtube_api_proxy", "str", "youtube", "youtube_api_proxy", allow_empty=True),
|
||||
# --- Google OAuth client (Sign in with Google; both encrypted, env fallback). Set from the
|
||||
# install wizard / Configuration page so no restart is needed when the creds change. ---
|
||||
ConfigSpec("google_client_id", "str", "google", "google_client_id", secret=True),
|
||||
ConfigSpec("google_client_secret", "str", "google", "google_client_secret", secret=True),
|
||||
# --- Access / registration ---
|
||||
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
|
||||
# --- Download center (yt-dlp) — per-user defaults + retention/GC + layout ---
|
||||
ConfigSpec("download_total_max_bytes", "int", "downloads", "download_total_max_bytes", min=0),
|
||||
ConfigSpec(
|
||||
@@ -137,7 +144,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
ConfigSpec(
|
||||
"download_gc_grace_days", "int", "downloads", "download_gc_grace_days", min=0, max=3_650
|
||||
),
|
||||
ConfigSpec("download_layout", "str", "downloads", "download_layout"),
|
||||
ConfigSpec("download_layout", "choice", "downloads", "download_layout"),
|
||||
ConfigSpec(
|
||||
"download_worker_concurrency",
|
||||
"int",
|
||||
@@ -150,6 +157,13 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
# --- Audit log ---
|
||||
# Days to keep admin audit-log rows before the audit_gc job prunes them (0 = keep forever).
|
||||
ConfigSpec("audit_retention_days", "int", "audit", "audit_retention_days", min=0, max=3650),
|
||||
# --- Diagnostics (E22) ---
|
||||
# Runtime log level for the API process. Applied without a restart — see routes/config.py and
|
||||
# main.py's lifespan.
|
||||
ConfigSpec("log_level", "choice", "diagnostics", "log_level"),
|
||||
# Days the log_events table keeps a row (the log_gc job prunes). No "0 = forever": stored logs
|
||||
# carry account-linked request history, and the privacy policy names a limit.
|
||||
ConfigSpec("log_retention_days", "int", "diagnostics", "log_retention_days", min=1, max=90),
|
||||
# --- Plex integration (optional; local-file playback + Plex metadata) ---
|
||||
ConfigSpec("plex_enabled", "bool", "plex", "plex_enabled"),
|
||||
ConfigSpec("plex_server_url", "str", "plex", "plex_server_url"),
|
||||
@@ -159,14 +173,16 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"),
|
||||
# (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.)
|
||||
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
|
||||
# Transcode re-encode tuning (E6 S1). hw = "auto"|"on"|"off" (free-text, like search_source); the
|
||||
# libx264 knobs apply only to the software fallback. crf 0–51 (x264 range); max_height 0 = source.
|
||||
ConfigSpec("plex_transcode_hw", "str", "plex", "plex_transcode_hw"),
|
||||
ConfigSpec("plex_transcode_preset", "str", "plex", "plex_transcode_preset"),
|
||||
# Transcode re-encode tuning (E6 S1). The libx264 knobs apply only to the software fallback.
|
||||
# crf 0–51 (x264 range); max_height 0 = source.
|
||||
ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw"),
|
||||
ConfigSpec("plex_transcode_preset", "choice", "plex", "plex_transcode_preset"),
|
||||
ConfigSpec("plex_transcode_crf", "int", "plex", "plex_transcode_crf", min=0, max=51),
|
||||
ConfigSpec(
|
||||
"plex_transcode_max_height", "int", "plex", "plex_transcode_max_height", min=0, max=4320
|
||||
),
|
||||
# Audio loudness normalisation on the HLS path (remux + transcode alike, not direct play).
|
||||
ConfigSpec("plex_audio_loudnorm", "bool", "plex", "plex_audio_loudnorm"),
|
||||
# NOTE: plex_watch_sync_interval_min is a config.py default read by the watch-sync scheduler job
|
||||
# (scheduler.py); it is intentionally NOT a runtime admin knob, so it stays out of SPECS.
|
||||
# --- TMDB enrichment (E5 S2; optional) — the API key is the only secret ---
|
||||
@@ -192,11 +208,23 @@ def _default(s: ConfigSpec):
|
||||
return getattr(settings, s.default_attr)
|
||||
|
||||
|
||||
def _choice(s: ConfigSpec, raw: object) -> str:
|
||||
"""The canonical spelling of `raw` among the key's choices (any letter case), or ValueError."""
|
||||
option = canonical_choice(s.choices, raw)
|
||||
if option is None:
|
||||
raise ValueError(f"{s.key} must be one of: {', '.join(s.choices)}")
|
||||
return option
|
||||
|
||||
|
||||
def _coerce(s: ConfigSpec, raw: str):
|
||||
if s.type == "int":
|
||||
return int(raw)
|
||||
if s.type == "bool":
|
||||
return raw == "true"
|
||||
if s.type == "choice":
|
||||
# A stored value that is no longer a choice (the list shrank) reads as the default, like a
|
||||
# non-numeric int override does — see get().
|
||||
return _choice(s, raw)
|
||||
return raw
|
||||
|
||||
|
||||
@@ -272,6 +300,8 @@ def set_value(db: Session, key: str, value) -> None:
|
||||
raw = str(v)
|
||||
elif s.type == "bool":
|
||||
raw = "true" if value else "false"
|
||||
elif s.type == "choice":
|
||||
raw = _choice(s, value)
|
||||
else:
|
||||
raw = "" if value is None else str(value)
|
||||
|
||||
@@ -314,6 +344,7 @@ def describe(db: Session) -> dict:
|
||||
"max": s.max,
|
||||
"is_set": is_set, # an override row exists
|
||||
"allow_empty": s.allow_empty, # if true, the UI stores "" rather than deleting the row
|
||||
"choices": list(s.choices), # the picker's options; [] for any other type
|
||||
}
|
||||
if s.secret:
|
||||
item["default_is_set"] = bool(_default(s))
|
||||
|
||||
+60
-15
@@ -28,16 +28,18 @@ from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app import logs
|
||||
from app import logsink
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import quota, service
|
||||
from app.downloads import credentials, formats, quota, service
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads.jobdb import (
|
||||
_edit_staging_dir,
|
||||
_requeue_if_running,
|
||||
_reset_asset_pending,
|
||||
_set_job,
|
||||
_staging_dir,
|
||||
)
|
||||
from app.downloads.status import STOPPED_BY_ROUTE, JobStatus
|
||||
@@ -253,7 +255,27 @@ def _fail_after_kill(
|
||||
"""`_stop_and_free` + mark the job errored — for the watchdog's stall/timeout kills and the
|
||||
supervisor-error fallback."""
|
||||
_stop_and_free(job_id, asset_id, proc, grace_s)
|
||||
_set_job(job_id, status="error", error=error, error_code=error_code, phase=None)
|
||||
_fail_running(job_id, error=error, error_code=error_code)
|
||||
|
||||
|
||||
def _fail_running(
|
||||
job_id: int, *, error: str, error_code: str | None, error_detail: str | None = None
|
||||
) -> None:
|
||||
"""Fail a claimed job — only while it is still `running` (`service.finalize_if_running`), so a
|
||||
cancel/pause a route landed first is never turned back into `error`, which holds the asset the
|
||||
route already released. `error_detail` is written every time: a failure never shows an earlier
|
||||
failure's explanation."""
|
||||
with SessionLocal() as db:
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=error,
|
||||
error_code=error_code,
|
||||
error_detail=error_detail,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
@@ -329,12 +351,10 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
status, _ = _job_liveness(job_id)
|
||||
if status == "running":
|
||||
_cleanup_staging(asset_id)
|
||||
_set_job(
|
||||
_fail_running(
|
||||
job_id,
|
||||
status="error",
|
||||
error_code="unknown",
|
||||
phase=None,
|
||||
error=f"The download process exited unexpectedly (code {proc.returncode}).",
|
||||
error_code="unknown",
|
||||
)
|
||||
log.warning(
|
||||
"job %s: child exited %s without finishing", job_id, proc.returncode
|
||||
@@ -399,16 +419,32 @@ def _process_job(job_id: int) -> None:
|
||||
asset_id = asset.id if asset else None
|
||||
asset_status = asset.status if asset else None
|
||||
asset_error = asset.error if asset else None
|
||||
if asset_status == "error":
|
||||
# A job that meets an already-failed asset gets that failure's explanation — not a bare
|
||||
# "Download failed". Only the text is on the asset, so it is diagnosed again, for this
|
||||
# link, with whether a cookie goes with it asked of the same gate the download child
|
||||
# used (`credentials.usable_cookie`) — as it stands now, so the advice fits this job's Retry.
|
||||
url = service.source_url(job.source_kind, job.source_ref)
|
||||
cookie = credentials.usable_cookie(
|
||||
db, job.user_id, url, asset, formats.normalize(job.profile_snapshot)
|
||||
)
|
||||
diagnosis = dlerrors.diagnose(asset_error, url=url, cookie_attached=cookie is not None)
|
||||
del cookie # only whether there is one matters here
|
||||
|
||||
if asset is None:
|
||||
_set_job(job_id, status="error", error="No media asset", phase=None)
|
||||
_fail_running(job_id, error="No media asset", error_code=None)
|
||||
return
|
||||
|
||||
if asset_status == "ready":
|
||||
_finish_from_cache(job_id, asset_id)
|
||||
return
|
||||
if asset_status == "error":
|
||||
_set_job(job_id, status="error", error=asset_error or "Download failed", phase=None)
|
||||
_fail_running(
|
||||
job_id,
|
||||
error=asset_error or "Download failed",
|
||||
error_code=diagnosis.code,
|
||||
error_detail=diagnosis.detail,
|
||||
)
|
||||
return
|
||||
if asset_status == "downloading":
|
||||
# Another worker won this asset between our claim and here (a narrow race — the claim SQL
|
||||
@@ -457,15 +493,20 @@ def _worker_loop(idx: int) -> None:
|
||||
if job_id is None:
|
||||
_stop.wait(settings.download_poll_seconds)
|
||||
continue
|
||||
try:
|
||||
_process_job(job_id)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("processing job %s crashed", job_id)
|
||||
_set_job(job_id, status="error", error="Worker error", phase=None)
|
||||
# Tag every line this job produces (including the crash below) with the job id: a worker runs
|
||||
# N jobs at once on N threads, so without it the interleaved output is unreadable. The bind
|
||||
# wraps the handler too — bound inside the `try`, it was already reset by the time the crash
|
||||
# line was logged.
|
||||
with logs.bind(job_id=job_id):
|
||||
try:
|
||||
_process_job(job_id)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("processing job %s crashed", job_id)
|
||||
_fail_running(job_id, error="Worker error", error_code=None)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logs.configure_logging("worker")
|
||||
signal.signal(signal.SIGTERM, _handle_signal)
|
||||
signal.signal(signal.SIGINT, _handle_signal)
|
||||
|
||||
@@ -481,6 +522,10 @@ def main() -> None:
|
||||
if _stop.is_set():
|
||||
return # asked to shut down while waiting — that IS a clean exit
|
||||
raise SystemExit(1)
|
||||
# After the wait, not before: the DB is reachable now. The wait checks `download_jobs`, not
|
||||
# `log_events`, so on the deploy that first adds the table the worker may start before the API
|
||||
# has migrated — those few seconds of batches fail and say so on stderr; nothing else breaks.
|
||||
logsink.start()
|
||||
_recover_orphans()
|
||||
n = max(1, settings.download_worker_concurrency)
|
||||
log.info("Download worker started (concurrency=%d, root=%s).", n, settings.download_root)
|
||||
|
||||
@@ -14,7 +14,14 @@ class RssError(Exception):
|
||||
|
||||
Deliberately NOT the same as an empty feed: a failed poll used to return `[]`, which the
|
||||
caller stamped as a successful `last_rss_at` — so a channel whose feed 404s permanently
|
||||
(deleted, renamed id) looked like it was being polled fine forever, and nothing surfaced it."""
|
||||
(deleted, renamed id) looked like it was being polled fine forever, and nothing surfaced it.
|
||||
|
||||
`reason` is a short stable code (`http_404`, `transport`) so a poll over many channels can report
|
||||
WHAT failed as counts instead of one line per channel."""
|
||||
|
||||
def __init__(self, message: str, reason: str):
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def fetch_channel_feed(channel_id: str) -> list[dict]:
|
||||
@@ -24,9 +31,12 @@ def fetch_channel_feed(channel_id: str) -> list[dict]:
|
||||
try:
|
||||
resp = httpx.get(url, timeout=20.0, headers={"User-Agent": "Siftlode/1.0"})
|
||||
except httpx.HTTPError as exc:
|
||||
raise RssError(f"RSS fetch failed for {channel_id}: {exc}") from exc
|
||||
raise RssError(f"RSS fetch failed for {channel_id}: {exc}", "transport") from exc
|
||||
if resp.status_code != 200:
|
||||
raise RssError(f"RSS fetch for {channel_id} returned HTTP {resp.status_code}")
|
||||
raise RssError(
|
||||
f"RSS fetch for {channel_id} returned HTTP {resp.status_code}",
|
||||
f"http_{resp.status_code}",
|
||||
)
|
||||
|
||||
parsed = feedparser.parse(resp.content)
|
||||
out: list[dict] = []
|
||||
|
||||
@@ -11,8 +11,12 @@ echo "Starting Siftlode API..."
|
||||
# on the next request -> 502. Non-idempotent POSTs (the player's progress/state writes,
|
||||
# fired every 5s) can't be auto-retried by the proxy, so those 502s reached the user.
|
||||
#
|
||||
# SECURITY: do NOT add --proxy-headers / --forwarded-allow-ips here. auth._client_ip trusts
|
||||
# SECURITY: do NOT add --proxy-headers / --forwarded-allow-ips here. ratelimit.client_key trusts
|
||||
# X-Forwarded-For only when the TRUE socket peer is a configured proxy (TRUSTED_PROXY_IPS); those
|
||||
# flags would make uvicorn overwrite request.client from the forgeable XFF, defeating that check and
|
||||
# reopening the rate-limit bypass (SA3).
|
||||
exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --log-config log_config.json --timeout-keep-alive 75
|
||||
#
|
||||
# No --log-config: app/logs.py configures logging for every Siftlode process (api, worker, download
|
||||
# child, alembic) in one place, and it silences uvicorn's access log in favour of our own
|
||||
# `http.request` line (route template, no query string, request id).
|
||||
exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --timeout-keep-alive 75
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
{
|
||||
"version": 1,
|
||||
"disable_existing_loggers": false,
|
||||
"formatters": {
|
||||
"default": {
|
||||
"()": "uvicorn.logging.DefaultFormatter",
|
||||
"fmt": "%(asctime)s %(levelprefix)s [%(name)s] %(message)s"
|
||||
},
|
||||
"access": {
|
||||
"()": "uvicorn.logging.AccessFormatter",
|
||||
"fmt": "%(asctime)s %(levelprefix)s %(client_addr)s - \"%(request_line)s\" %(status_code)s"
|
||||
},
|
||||
"plain": {
|
||||
"format": "%(asctime)s %(levelname)-5s [%(name)s] %(message)s"
|
||||
}
|
||||
},
|
||||
"handlers": {
|
||||
"default": {
|
||||
"class": "logging.StreamHandler",
|
||||
"formatter": "default",
|
||||
"stream": "ext://sys.stdout"
|
||||
},
|
||||
"access": {
|
||||
"class": "logging.StreamHandler",
|
||||
"formatter": "access",
|
||||
"stream": "ext://sys.stdout"
|
||||
},
|
||||
"plain": {
|
||||
"class": "logging.StreamHandler",
|
||||
"formatter": "plain",
|
||||
"stream": "ext://sys.stdout"
|
||||
}
|
||||
},
|
||||
"loggers": {
|
||||
"uvicorn": { "handlers": ["default"], "level": "INFO", "propagate": false },
|
||||
"uvicorn.error": { "handlers": ["default"], "level": "INFO", "propagate": false },
|
||||
"uvicorn.access": { "handlers": ["access"], "level": "INFO", "propagate": false },
|
||||
"siftlode": { "handlers": ["plain"], "level": "INFO", "propagate": false }
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
# Test + lint tooling — NEVER in the prod image. Installed only by backend/Dockerfile.test (the
|
||||
# throwaway test image), which the prod build (root Dockerfile) never references. Pinned so the
|
||||
# suite is reproducible: pytest for the pure-logic tests, and ruff at the SAME version the gate's
|
||||
# backend lint lane currently runs on the host (see the run_lint note in siftlode.sh). httpx is
|
||||
# already an app dependency (requirements.txt), so its TestClient needs nothing extra here.
|
||||
# backend lint lane runs on the host (the gate fails when the host's ruff is not this one). httpx2:
|
||||
# Starlette 1.x's TestClient is built on it (`import httpx2 as httpx`) and only falls back to the
|
||||
# app's own httpx with a deprecation warning on every run.
|
||||
pytest==8.4.2
|
||||
ruff==0.15.21
|
||||
httpx2==2.13.0
|
||||
|
||||
@@ -7,6 +7,7 @@ Authlib==1.7.2
|
||||
Mako==1.4.1
|
||||
MarkupSafe==3.0.3
|
||||
PyYAML==6.0.3
|
||||
Pygments==2.21.0
|
||||
SQLAlchemy==2.0.52
|
||||
alembic==1.19.1
|
||||
annotated-doc==0.0.5
|
||||
@@ -21,6 +22,7 @@ cffi==2.1.1
|
||||
charset-normalizer==3.5.1
|
||||
click==8.4.2
|
||||
cryptography==50.0.0
|
||||
curl_cffi==0.15.0
|
||||
fastapi==0.141.1
|
||||
feedparser-sgmllib==2.1.0
|
||||
feedparser==6.0.14
|
||||
@@ -32,6 +34,8 @@ httpx==0.28.1
|
||||
idna==3.18
|
||||
itsdangerous==2.2.0
|
||||
joserfc==1.7.4
|
||||
markdown-it-py==4.2.0
|
||||
mdurl==0.1.2
|
||||
mutagen==1.48.1
|
||||
numpy==2.5.2
|
||||
psycopg-binary==3.3.4
|
||||
@@ -45,6 +49,7 @@ pydantic_core==2.46.4
|
||||
python-dotenv==1.2.3
|
||||
python-multipart==0.0.32
|
||||
requests==2.34.2
|
||||
rich==15.0.0
|
||||
starlette==1.6.0
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
|
||||
@@ -21,6 +21,13 @@ argon2-cffi>=23.1,<26
|
||||
# scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so
|
||||
# keep the floor loose and bump it when extraction breaks.
|
||||
yt-dlp[default]>=2025.5.22
|
||||
# Browser impersonation (`impersonate`) for Facebook downloads made with the user's cookie: some
|
||||
# links answer a signed-in plain request with "Cannot parse data" and only parse as Chrome (E11.5,
|
||||
# measured). The range is YT-DLP's, not ours — yt-dlp 2026.07.04 imports curl_cffi only in
|
||||
# 0.10.x–0.15.x and silently reports every target "unavailable" outside it. Move it when bumping
|
||||
# yt-dlp; `tests/test_facebook_impersonation.py::test_the_locked_curl_cffi_can_impersonate_chrome`
|
||||
# fails the gate if the locked pair can no longer impersonate.
|
||||
curl_cffi>=0.10,<0.16
|
||||
# PO Token provider plugin: talks to the bgutil sidecar (brainicism/bgutil-ytdlp-pot-provider)
|
||||
# to mint YouTube Proof-of-Origin tokens on demand, bypassing "confirm you're not a bot" and
|
||||
# unlocking high-quality formats. Pin to match the sidecar image tag (versions must align).
|
||||
|
||||
@@ -57,6 +57,10 @@ def _db_reachable() -> bool:
|
||||
|
||||
_DB_REACHABLE = _db_reachable()
|
||||
|
||||
# No background log writer in tests: every TestClient would otherwise start one on app startup and
|
||||
# write into tables the fixtures truncate between tests. The sink's own tests drive it explicitly.
|
||||
settings.log_db_sink = False
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def _schema():
|
||||
@@ -210,3 +214,15 @@ def _reset_rate_limiters():
|
||||
for obj in vars(mod).values():
|
||||
if isinstance(obj, RateLimiter):
|
||||
obj.reset()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_log_erasures():
|
||||
"""`logs._erased` is module-level and remembers an erased account for an hour of monotonic time
|
||||
— longer than any suite run — while the `db` fixture hands serial ids back out from 1 via
|
||||
TRUNCATE … RESTART IDENTITY. So a test that purges user 1 would otherwise make a LATER test's
|
||||
rows for user 1 store with `user_id = None`, failing somewhere nowhere near the cause and only
|
||||
in some file orders. Same reasoning as the limiter sweep above, different global."""
|
||||
from app import logs
|
||||
|
||||
logs._erased.clear()
|
||||
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
{
|
||||
"_comment": "One redaction contract for both logs: backend/app/logs.py redact() and frontend/src/lib/log.ts redact() must turn every `value` (under `key`) into `expected`. Both test suites run this file (backend tests/test_logs.py, frontend src/lib/log.contract.test.ts), so the two implementations cannot drift apart. Some cases prove a secret goes; others prove ordinary content (a search term, a Plex title, a chat line) is not mangled — what a LEVEL may carry is decided at the call site, not here.",
|
||||
"cases": [
|
||||
{
|
||||
"name": "a password field is dropped whole",
|
||||
"key": "",
|
||||
"value": {
|
||||
"email": "a@b.co",
|
||||
"password": "hunter2",
|
||||
"current_password": "x"
|
||||
},
|
||||
"expected": {
|
||||
"email": "[email]",
|
||||
"password": "[redacted]",
|
||||
"current_password": "[redacted]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a token under any spelling of the key, nested and in lists",
|
||||
"key": "",
|
||||
"value": {
|
||||
"plex": { "plex_server_token": "abc", "Authorization": "Bearer z" },
|
||||
"items": [{ "access_token": "t1" }, { "id": 3 }]
|
||||
},
|
||||
"expected": {
|
||||
"plex": {
|
||||
"plex_server_token": "[redacted]",
|
||||
"Authorization": "[redacted]"
|
||||
},
|
||||
"items": [{ "access_token": "[redacted]" }, { "id": 3 }]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "E2EE material and cookies",
|
||||
"key": "",
|
||||
"value": {
|
||||
"ciphertext": "AAAA",
|
||||
"passphrase": "p",
|
||||
"private_key": "k",
|
||||
"cookies": "SID=1"
|
||||
},
|
||||
"expected": {
|
||||
"ciphertext": "[redacted]",
|
||||
"passphrase": "[redacted]",
|
||||
"private_key": "[redacted]",
|
||||
"cookies": "[redacted]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a reset link and a verify fragment",
|
||||
"key": "msg",
|
||||
"value": "open https://app.example/#reset=QWErty123 or /?verify=abc&lang=hu",
|
||||
"expected": "open https://app.example/#reset=[redacted] or /?verify=[redacted]&lang=hu"
|
||||
},
|
||||
{
|
||||
"name": "an OAuth callback URL",
|
||||
"key": "url",
|
||||
"value": "/auth/callback?code=4/0Ab-xyz&state=s3cr3t&scope=email",
|
||||
"expected": "/auth/callback?code=[redacted]&state=[redacted]&scope=email"
|
||||
},
|
||||
{
|
||||
"name": "an email address inside a sentence",
|
||||
"key": "message",
|
||||
"value": "Invitation sent to Some.One+tag@mail.example.org.",
|
||||
"expected": "Invitation sent to [email]."
|
||||
},
|
||||
{
|
||||
"name": "a search term is kept (TRACE decides whether it is logged at all)",
|
||||
"key": "q",
|
||||
"value": "lo-fi beats to relax",
|
||||
"expected": "lo-fi beats to relax"
|
||||
},
|
||||
{
|
||||
"name": "a Plex title with an at-sign is not an address",
|
||||
"key": "title",
|
||||
"value": "Live @ Wembley 1986",
|
||||
"expected": "Live @ Wembley 1986"
|
||||
},
|
||||
{
|
||||
"name": "a chat line is text like any other; its only secret part is scrubbed",
|
||||
"key": "body",
|
||||
"value": "see you at 8, my mail is me@home.example",
|
||||
"expected": "see you at 8, my mail is [email]"
|
||||
},
|
||||
{
|
||||
"name": "non-text values pass through",
|
||||
"key": "",
|
||||
"value": { "status": 404, "ok": false, "ratio": 0.5, "none": null },
|
||||
"expected": { "status": 404, "ok": false, "ratio": 0.5, "none": null }
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Per-source adapter selection + the source-specific policy each one carries. These lock the
|
||||
behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch."""
|
||||
|
||||
from app.config import settings
|
||||
from app.titles import normalize_title
|
||||
from app.downloads.adapters import (
|
||||
FacebookAdapter,
|
||||
RedditAdapter,
|
||||
@@ -103,12 +105,74 @@ def test_generic_source_url_is_passthrough_youtube_reconstructs():
|
||||
)
|
||||
|
||||
|
||||
def test_client_sets_and_retriable_are_youtube_only():
|
||||
def test_client_sets_and_retriable_are_per_source():
|
||||
# Generic sources make a single attempt and never retry on the YouTube per-client markers.
|
||||
assert SourceAdapter().client_sets() == [None]
|
||||
assert SourceAdapter().client_sets(authed=True) == [None]
|
||||
assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False
|
||||
assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True
|
||||
assert YouTubeAdapter().retriable("some unrelated network blip") is False
|
||||
# A client list yt-dlp empties because none of its clients take cookies does NOT raise "No
|
||||
# player clients have been requested" (that raise sits earlier in yt-dlp, before the
|
||||
# cookie-capability removal). Measured with a real cookie: the emptied request fails with
|
||||
# "Requested format is not available", which is retriable here already — so the wasted attempt
|
||||
# is recoverable and `_COOKIE_CAPABLE` exists to avoid spending it at all.
|
||||
assert (
|
||||
YouTubeAdapter().retriable(
|
||||
"ERROR: [youtube] abc: Requested format is not available. Use --list-formats"
|
||||
)
|
||||
is True
|
||||
)
|
||||
# The media fetch refused after a successful extraction: the format URLs belong to the client
|
||||
# that minted them, so the next client set is a real second chance (it used to end the job).
|
||||
assert (
|
||||
YouTubeAdapter().retriable(
|
||||
"ERROR: unable to download video data: HTTP Error 403: Forbidden"
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
SourceAdapter().retriable("ERROR: unable to download video data: HTTP Error 403") is False
|
||||
)
|
||||
# Facebook: a second, Chrome-impersonating attempt only WITH the user's cookie, and only behind
|
||||
# the parse wall (without a cookie that wall is the login wall — no disguise gets past it).
|
||||
fb = FacebookAdapter()
|
||||
assert fb.client_sets() == [None]
|
||||
assert len(fb.client_sets(authed=True)) == 2
|
||||
assert fb.ydl_opts_extra(fb.client_sets(authed=True)[0]) == {}
|
||||
assert fb.retriable("ERROR: [facebook] 1: Cannot parse data; please report this issue") is True
|
||||
assert fb.retriable("ERROR: [facebook] 1: No video formats found!") is False
|
||||
|
||||
|
||||
def test_a_signed_in_youtube_download_only_runs_cookie_capable_clients(monkeypatch):
|
||||
yt = YouTubeAdapter()
|
||||
monkeypatch.setattr(settings, "download_player_clients", "android_vr")
|
||||
monkeypatch.setattr(settings, "download_player_clients_fallback", "web_safari,web,android")
|
||||
# Signed out: the admin's ladder exactly as configured.
|
||||
assert yt.client_sets() == [["android_vr"], ["web_safari", "web", "android"]]
|
||||
# Signed in: yt-dlp REMOVES every client that takes no cookies, and a set it empties that way
|
||||
# raises "No player clients have been requested" — which would fail the download on attempt 1,
|
||||
# before the POT-backed web fallback the cookie exists for. So the adapter never asks for one:
|
||||
# the android_vr-only set is gone and `android` is dropped out of the fallback.
|
||||
assert yt.client_sets(authed=True) == [["web_safari", "web"]]
|
||||
|
||||
|
||||
def test_a_signed_in_download_still_has_clients_when_the_admin_configured_none(monkeypatch):
|
||||
# Both knobs cookie-incapable: fall back to the POT-backed web pair rather than no attempt at all.
|
||||
monkeypatch.setattr(settings, "download_player_clients", "android_vr")
|
||||
monkeypatch.setattr(settings, "download_player_clients_fallback", "android,ios")
|
||||
assert YouTubeAdapter().client_sets(authed=True) == [["web_safari", "web"]]
|
||||
|
||||
|
||||
def test_the_cookie_capable_clients_match_the_installed_ytdlp():
|
||||
# The list is yt-dlp's, not ours (INNERTUBE_CLIENTS[...]["SUPPORTS_COOKIES"]). Pinned here so a
|
||||
# yt-dlp bump that renames or re-flags a client turns the gate red, instead of silently sending
|
||||
# a signed-in download to a client that drops the cookie.
|
||||
from yt_dlp.extractor.youtube._base import INNERTUBE_CLIENTS
|
||||
|
||||
live = {k for k, v in INNERTUBE_CLIENTS.items() if v.get("SUPPORTS_COOKIES")}
|
||||
assert live == set(YouTubeAdapter._COOKIE_CAPABLE)
|
||||
assert set(YouTubeAdapter._AUTHED_DEFAULT) <= live
|
||||
|
||||
|
||||
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
|
||||
@@ -130,3 +194,28 @@ def test_wants_metadata_backfill_matches_old_should_refetch():
|
||||
def test_nfo_uniqueid_type_is_the_adapter_key():
|
||||
assert YouTubeAdapter().nfo_uniqueid_type() == "youtube"
|
||||
assert SourceAdapter().nfo_uniqueid_type() == "generic"
|
||||
|
||||
|
||||
def test_title_is_yt_dlps_for_a_source_without_placeholders():
|
||||
assert SourceAdapter().title({"title": "Video", "description": "text"}) == "Video"
|
||||
assert SourceAdapter().title({}) is None
|
||||
|
||||
|
||||
def test_facebook_placeholder_title_takes_the_posts_text():
|
||||
# Measured: signed in, yt-dlp titles every reel "Video" (an 18+ reel read as Chrome: "Facebook"),
|
||||
# while the post's text is still in the description.
|
||||
a = FacebookAdapter()
|
||||
info = {"title": "Video", "description": "\n Tune of The Day\nsecond line"}
|
||||
assert a.title(info) == "Tune of The Day"
|
||||
assert a.title({**info, "title": "Facebook"}) == "Tune of The Day"
|
||||
# A real title (signed out) is left for normalize_title, engagement prefix and all.
|
||||
assert a.title({**info, "title": "456K views | Tune"}) == "456K views | Tune"
|
||||
# Nothing better on the page: keep what yt-dlp gave.
|
||||
assert a.title({"title": "Facebook", "description": None}) == "Facebook"
|
||||
|
||||
|
||||
def test_facebook_title_from_a_long_description_is_the_whole_line_after_cleaning():
|
||||
# Dev UAT: a cut with "…" came out of normalize_title as "… live at." (NFKC "..." collapsed to "."),
|
||||
# a sentence that seemed to end. The whole first line survives the cleaning intact instead.
|
||||
line = 'Tune of The Day, Argy playing his version of the classic "Resurrection" with Paul Oakenfold live at Space'
|
||||
assert normalize_title(FacebookAdapter().title({"title": "Video", "description": line})) == line
|
||||
|
||||
@@ -12,6 +12,7 @@ Note: `/api/me` is the bootstrap probe — it returns 200 `{authenticated: False
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.auth import ACTIVE_ACCOUNT_HEADER
|
||||
@@ -57,6 +58,75 @@ def test_a_non_wallet_account_header_falls_back_to_the_default(client, make_user
|
||||
assert me.json()["email"] == "alice2@example.com" # still Alice, not Mallory
|
||||
|
||||
|
||||
def test_a_tab_pinned_to_an_account_signed_out_elsewhere_acts_as_nobody(client, make_user):
|
||||
# Two accounts in one browser: a tab pinned to Alice, Bob the default. Alice signs out in another
|
||||
# tab. The pinned tab used to go on acting as the default, Bob — its player checkpoints and pref
|
||||
# saves landed in his account, and its own Sign out signed HIM out.
|
||||
alice = make_user(email="alice3@example.com", password=_PW)
|
||||
make_user(email="bob3@example.com", password=_PW)
|
||||
assert _login(client, "alice3@example.com", _PW).status_code == 200
|
||||
assert _login(client, "bob3@example.com", _PW).status_code == 200 # Bob is now the default
|
||||
tab = {ACTIVE_ACCOUNT_HEADER: str(alice.id)}
|
||||
assert client.get("/api/feed", headers=tab).status_code == 200
|
||||
assert client.post("/auth/logout", headers=tab).status_code == 200 # Alice signs out
|
||||
|
||||
assert client.get("/api/feed", headers=tab).status_code == 401
|
||||
assert client.post("/auth/logout", headers=tab).status_code == 200
|
||||
assert client.get("/api/feed").status_code == 200 # Bob is still signed in, the session intact
|
||||
# The bootstrap probe still answers as the default: how the stale tab learns who it is now.
|
||||
assert client.get("/api/me", headers=tab).json()["email"] == "bob3@example.com"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ended", ["revoked", "suspended"])
|
||||
def test_a_tab_pinned_to_an_account_whose_session_ended_comes_back_as_the_default(
|
||||
client, make_user, db, ended
|
||||
):
|
||||
# A tab pinned to Alice, Bob the default. Alice's session then ends while she is still in this
|
||||
# browser's wallet ("log out everywhere" on another device, or an admin suspends her). The tab's
|
||||
# bootstrap probe used to answer "signed out": the landing page with Bob still signed in, and no way
|
||||
# off it but closing the tab (signing in as anyone else there kept the tab pinned to Alice).
|
||||
alice = make_user(email="alice4@example.com", password=_PW)
|
||||
make_user(email="bob4@example.com", password=_PW)
|
||||
assert _login(client, "alice4@example.com", _PW).status_code == 200
|
||||
assert _login(client, "bob4@example.com", _PW).status_code == 200
|
||||
tab = {ACTIVE_ACCOUNT_HEADER: str(alice.id)}
|
||||
assert client.get("/api/feed", headers=tab).status_code == 200
|
||||
if ended == "revoked":
|
||||
alice.session_epoch = (alice.session_epoch or 0) + 1
|
||||
else:
|
||||
alice.is_suspended = True
|
||||
db.commit()
|
||||
|
||||
assert client.get("/api/me", headers=tab).json()["email"] == "bob4@example.com"
|
||||
assert client.get("/api/feed", headers=tab).status_code == 401 # still nobody, never Bob
|
||||
assert client.get("/api/feed").status_code == 200 # Bob's session is intact
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ended", ["revoked", "suspended", "deactivated"])
|
||||
def test_switching_to_an_account_whose_session_ended_refuses_it_and_drops_it_from_the_list(
|
||||
client, make_user, db, monkeypatch, ended
|
||||
):
|
||||
# The account switcher lists this browser's wallet. A revoked account was dropped from it on a refused
|
||||
# switch, but a suspended one stayed, refused on every click; a deactivated one was not refused at all —
|
||||
# made the default, the next request found it inactive and signed every account here out.
|
||||
monkeypatch.setattr("app.routes.me.is_allowed", lambda _db, _email: True) # both hold an invite
|
||||
alice = make_user(email="alice5@example.com", password=_PW)
|
||||
make_user(email="bob5@example.com", password=_PW)
|
||||
assert _login(client, "alice5@example.com", _PW).status_code == 200
|
||||
assert _login(client, "bob5@example.com", _PW).status_code == 200
|
||||
if ended == "revoked":
|
||||
alice.session_epoch = (alice.session_epoch or 0) + 1
|
||||
elif ended == "suspended":
|
||||
alice.is_suspended = True
|
||||
else:
|
||||
alice.is_active = False
|
||||
db.commit()
|
||||
|
||||
assert client.post("/api/me/switch", json={"user_id": alice.id}).status_code == 403
|
||||
assert [a["email"] for a in client.get("/api/me/accounts").json()] == ["bob5@example.com"]
|
||||
assert client.get("/api/feed").status_code == 200 # Bob is still signed in
|
||||
|
||||
|
||||
def test_logout_others_revokes_the_other_session_but_keeps_the_caller(client, make_user):
|
||||
# Two independent cookie jars (browsers) signed into the same account. `logout-others` from one
|
||||
# bumps the account epoch so the OTHER's cookie is rejected on its next request, while the caller
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""The shared demo account's refusal says WHY, in the reader's language (deep-review round 6).
|
||||
|
||||
`require_human` answered `403 "Not available in the demo account."` — a plain string. `req` leaves
|
||||
403 to the caller, and round 5 taught `notifyActionError` to discard a plain-string detail in favour
|
||||
of the caller's already-translated fallback, so from that commit on the demo user got a generic
|
||||
"couldn't do that" and no hint that the demo account was the reason. Not every affordance behind
|
||||
this gate is UI-gated on `is_demo` either — the comfort shelf is not, so the pin button is live.
|
||||
|
||||
A structured `{code: "demo"}` is what `localizeDetail` can turn into `errors.demo` in both locales.
|
||||
Pinned at every site: the shared dependency, the two auth routes that hand-roll the same check,
|
||||
the account-delete route in `routes/me.py` — which round 6 missed while its docstring claimed
|
||||
all the sites were covered, so the next reader had no reason to look (round 7). An asymmetric fix is
|
||||
the smell this repo bans, and a docstring asserting symmetry it does not have is worse than none.
|
||||
|
||||
The list is derived from the source, not typed out: `test_every_is_demo_refusal_is_on_the_list`
|
||||
below fails when a fifth `is_demo` 403 appears anywhere in `app/`, so this file cannot fall behind
|
||||
the code the way round 6's did. (E22 S5 added the fifth: raising the diagnostics level.) Needs the DB.
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from app.models import User
|
||||
|
||||
|
||||
def test_every_demo_refusal_carries_the_same_structured_reason(db, make_user, as_user):
|
||||
demo = make_user(email="demo@example.com", is_demo=True)
|
||||
client = as_user(demo)
|
||||
for method, url, body in [
|
||||
# `require_human` itself, through an affordance the UI does NOT gate on is_demo.
|
||||
("post", "/api/discover/pins", {"kind": "video", "ref": "v1"}),
|
||||
# The three routes that repeat the check inline rather than depending on the gate.
|
||||
("post", "/auth/logout-others", None),
|
||||
("post", "/auth/set-password", {"password": "a long enough password"}),
|
||||
("delete", "/api/me/account", None),
|
||||
# E22 S5: the shared account's log would interleave every visitor's session.
|
||||
("put", "/api/me/diagnostics", {"level": "DEBUG"}),
|
||||
]:
|
||||
r = getattr(client, method)(url, **({"json": body} if body is not None else {}))
|
||||
assert r.status_code == 403, (url, r.text)
|
||||
detail = r.json()["detail"]
|
||||
# Every one of them is a dict whose `code` the frontend can localize; the delete route adds
|
||||
# a `reason` so its refusal can be worded specifically.
|
||||
assert isinstance(detail, dict), (url, detail)
|
||||
assert detail["code"] == "demo", (url, detail)
|
||||
|
||||
|
||||
def test_every_is_demo_refusal_is_on_the_list():
|
||||
"""A fifth site must not be able to appear unnoticed.
|
||||
|
||||
Round 6 converted three of four `is_demo` 403s and wrote a docstring saying it had converted
|
||||
them all; the fourth kept answering a bare English string for another round because nothing
|
||||
counted them. So count them here: every `is_demo`-guarded 403 in `app/` has to raise a
|
||||
structured detail, and the number of them has to match what this file exercises."""
|
||||
app_dir = Path(__file__).resolve().parents[1] / "app"
|
||||
sites = []
|
||||
for path in sorted(app_dir.rglob("*.py")):
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
for i, line in enumerate(lines):
|
||||
# `if user.is_demo:` / `if u.is_demo and …:` — the hand-rolled gate, not the ORM
|
||||
# filters (`User.is_demo.is_(False)`) and not the column definition.
|
||||
if not re.search(r"^\s*if\s+\w+\.is_demo\b[^:]*:", line):
|
||||
continue
|
||||
block = "\n".join(lines[i : i + 6])
|
||||
if "403" not in block:
|
||||
continue
|
||||
sites.append(f"{path.relative_to(app_dir)}:{i + 1}")
|
||||
assert 'detail={"code": "demo"' in block, (
|
||||
f"{path.relative_to(app_dir)}:{i + 1} refuses the demo account with an "
|
||||
f"unstructured detail — the frontend cannot localize it:\n{block}"
|
||||
)
|
||||
# auth.py's shared gate + routes/auth.py's two + routes/me.py's delete and diagnostics level. A
|
||||
# new one is welcome, but it has to be added to the loop above in the same commit.
|
||||
assert len(sites) == 5, sites
|
||||
|
||||
|
||||
def test_a_real_account_is_not_refused_by_the_gate(db, make_user, as_user):
|
||||
"""The other half: the structured detail must not come from having broken the gate open the
|
||||
other way. A non-demo account gets past `require_human` and fails (or succeeds) on its own
|
||||
merits — anything but a 403 demo refusal."""
|
||||
client = as_user(make_user(email="real@example.com"))
|
||||
r = client.post("/auth/logout-others")
|
||||
assert r.status_code == 200, r.text
|
||||
assert not isinstance(r.json().get("detail"), dict)
|
||||
|
||||
|
||||
def test_the_gate_reads_the_flag_and_not_the_email(db):
|
||||
"""`is_demo` is a column, not a convention about the address — a real account that happens to
|
||||
be called demo@ must not be refused."""
|
||||
from app.auth import require_human
|
||||
|
||||
assert require_human(User(email="demo@example.com", is_demo=False)) is not None
|
||||
@@ -0,0 +1,492 @@
|
||||
"""E22 S5 — a user's consented diagnostics level, and the browser log it lets in.
|
||||
|
||||
Three parts:
|
||||
* the level itself: the server computes its expiry, a lapsed one reads as the base, the demo
|
||||
account cannot raise it (DB lane, through the real routes);
|
||||
* `POST /api/diagnostics/events`: refused at the base level, and nothing in a batch is trusted —
|
||||
the account comes from the session, everything is redacted again, events under the consented
|
||||
level are dropped, sizes and pace are capped (DB lane);
|
||||
* the server side of the elevation: ONE user's requests are logged in detail while everyone
|
||||
else's stay at the base level, and the account reaches a handler's own lines at all (pure, an
|
||||
in-process app).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from fastapi import Depends, FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import diagnostics, logs, logsink, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import LogEvent, User
|
||||
from app.routes import diagnostics as diagnostics_routes
|
||||
|
||||
_PW = "correct horse battery"
|
||||
|
||||
|
||||
# --- the level ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_raised_level_counts_until_it_lapses_and_not_a_moment_longer():
|
||||
now = datetime.now(timezone.utc)
|
||||
assert diagnostics.effective_level(User()) == "INFO"
|
||||
raised = User(diagnostics_level="TRACE", diagnostics_until=now + timedelta(minutes=1))
|
||||
assert diagnostics.effective_level(raised, now) == "TRACE"
|
||||
assert diagnostics.effective_level(raised, now + timedelta(minutes=2)) == "INFO"
|
||||
# A level without an expiry is not a level that never ends — it is no level at all.
|
||||
assert diagnostics.effective_level(User(diagnostics_level="DEBUG")) == "INFO"
|
||||
|
||||
|
||||
def _signed_in(client, make_user, **kw):
|
||||
"""A real sign-in: `/api/me` resolves through `optional_current_user`, which the `as_user`
|
||||
override does not cover (see test_me_email_flag)."""
|
||||
user = make_user(email="diag@example.com", password=_PW, **kw)
|
||||
r = client.post("/auth/password-login", json={"email": "diag@example.com", "password": _PW})
|
||||
assert r.status_code == 200, r.text
|
||||
return user
|
||||
|
||||
|
||||
def test_raising_the_level_sets_the_servers_expiry_whatever_the_client_sends(db, client, make_user):
|
||||
user = _signed_in(client, make_user)
|
||||
r = client.put("/api/me/diagnostics", json={"level": "DEBUG", "until": "2099-01-01T00:00:00Z"})
|
||||
assert r.status_code == 200, r.text
|
||||
db.refresh(user)
|
||||
expected = datetime.now(timezone.utc) + timedelta(hours=diagnostics.ELEVATION_HOURS)
|
||||
assert abs((user.diagnostics_until - expected).total_seconds()) < 60
|
||||
assert r.json()["diagnostics"]["level"] == "DEBUG"
|
||||
assert client.get("/api/me").json()["diagnostics"]["level"] == "DEBUG"
|
||||
|
||||
assert client.put("/api/me/diagnostics", json={"level": "INFO"}).status_code == 200
|
||||
db.refresh(user)
|
||||
assert (user.diagnostics_level, user.diagnostics_until) == (None, None)
|
||||
assert client.get("/api/me").json()["diagnostics"] == {
|
||||
"level": "INFO",
|
||||
"until": None,
|
||||
"retention_days": 7,
|
||||
}
|
||||
|
||||
|
||||
def test_a_lapsed_level_reads_as_the_base_in_me(db, client, make_user):
|
||||
_signed_in(
|
||||
client,
|
||||
make_user,
|
||||
diagnostics_level="TRACE",
|
||||
diagnostics_until=datetime.now(timezone.utc) - timedelta(minutes=1),
|
||||
)
|
||||
assert client.get("/api/me").json()["diagnostics"] == {
|
||||
"level": "INFO",
|
||||
"until": None,
|
||||
"retention_days": 7,
|
||||
}
|
||||
|
||||
|
||||
def test_only_the_three_levels_can_be_chosen(db, make_user, as_user):
|
||||
client = as_user(make_user())
|
||||
for bogus in ("WARNING", "debug", "EVERYTHING"):
|
||||
assert client.put("/api/me/diagnostics", json={"level": bogus}).status_code == 422
|
||||
|
||||
|
||||
def test_the_demo_account_can_return_to_the_base_but_never_raise_it(db, make_user, as_user):
|
||||
client = as_user(make_user(is_demo=True))
|
||||
r = client.put("/api/me/diagnostics", json={"level": "TRACE"})
|
||||
assert r.status_code == 403 and r.json()["detail"]["code"] == "demo"
|
||||
assert client.put("/api/me/diagnostics", json={"level": "INFO"}).status_code == 200
|
||||
|
||||
|
||||
def test_preferences_must_be_a_json_object(db, make_user, as_user):
|
||||
client = as_user(make_user())
|
||||
assert client.put("/api/me/preferences", json=["not", "an", "object"]).status_code == 422
|
||||
r = client.put("/api/me/preferences", json={"theme": "dark"})
|
||||
assert r.status_code == 200 and r.json()["preferences"]["theme"] == "dark"
|
||||
|
||||
|
||||
# --- the browser's batches ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _event(**over) -> dict:
|
||||
return {
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"level": "INFO",
|
||||
"event": "nav.view",
|
||||
"msg": "view",
|
||||
"tab_id": "a1b2c3",
|
||||
"version": "0.88.0",
|
||||
**over,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def elevated(db, make_user, as_user):
|
||||
"""A signed-in client whose user consented to `level` a minute ago, and a reader for its rows."""
|
||||
diagnostics_routes._batches.reset()
|
||||
|
||||
def _make(level: str = "DEBUG"):
|
||||
user = make_user(
|
||||
diagnostics_level=level,
|
||||
diagnostics_until=datetime.now(timezone.utc) + timedelta(hours=23),
|
||||
)
|
||||
|
||||
def rows() -> list[LogEvent]:
|
||||
db.expire_all()
|
||||
return list(db.execute(select(LogEvent).order_by(LogEvent.id)).scalars())
|
||||
|
||||
return user, as_user(user), rows
|
||||
|
||||
yield _make
|
||||
diagnostics_routes._batches.reset()
|
||||
|
||||
|
||||
def test_nothing_is_accepted_at_the_base_level(db, make_user, as_user):
|
||||
client = as_user(make_user())
|
||||
r = client.post("/api/diagnostics/events", json={"events": [_event()]})
|
||||
assert r.status_code == 409 and r.json()["detail"]["code"] == "diagnostics_off"
|
||||
assert not db.execute(select(LogEvent)).first()
|
||||
|
||||
|
||||
def test_an_anonymous_caller_is_refused(client):
|
||||
assert client.post("/api/diagnostics/events", json={"events": [_event()]}).status_code == 401
|
||||
|
||||
|
||||
def test_a_batch_is_stored_as_the_sessions_web_events_and_redacted_again(elevated):
|
||||
user, client, rows = elevated("DEBUG")
|
||||
r = client.post(
|
||||
"/api/diagnostics/events",
|
||||
json={
|
||||
"events": [
|
||||
_event(
|
||||
level="WARNING",
|
||||
event="ui.notify",
|
||||
msg="sent to someone@example.com",
|
||||
attrs={"password": "hunter2", "url": "/?token=abc123", "user_id": 999},
|
||||
request_id="0123456789abcdef",
|
||||
),
|
||||
_event(level="DEBUG", event="ui.click", attrs={"control": "feed.refresh"}),
|
||||
],
|
||||
"user_id": 999, # ignored: the account is the session's
|
||||
},
|
||||
)
|
||||
assert r.json() == {"ok": True, "stored": 2}
|
||||
warning, click = rows()
|
||||
assert (warning.source, warning.logger, warning.user_id) == ("web", "web", user.id)
|
||||
assert warning.msg == "sent to [email]"
|
||||
assert warning.attrs["password"] == logs.REDACTED
|
||||
assert "abc123" not in json.dumps(warning.attrs)
|
||||
assert warning.attrs["tab_id"] == "a1b2c3"
|
||||
assert warning.request_id == "0123456789abcdef"
|
||||
assert (warning.sev, click.level, click.version) == (13, "DEBUG", "0.88.0")
|
||||
|
||||
|
||||
def test_an_erased_accounts_batch_is_stored_without_the_account(elevated):
|
||||
"""The sink's writer is not the only path into `log_events` — this route inserts too, and an
|
||||
account erased while its browser was mid-flush must not get its rows back through it. Faked at
|
||||
the `forget_user` seam the purge itself calls: once the User row is actually gone the session
|
||||
no longer resolves, so the reachable window is exactly a request already past `current_user`.
|
||||
(`logs._erased` is cleared before every test by conftest's `_reset_log_erasures`.)"""
|
||||
user, client, rows = elevated("DEBUG")
|
||||
logs.forget_user(user.id)
|
||||
r = client.post("/api/diagnostics/events", json={"events": [_event(level="WARNING")]})
|
||||
assert r.json() == {"ok": True, "stored": 1}
|
||||
(stored,) = rows()
|
||||
assert (stored.source, stored.user_id) == ("web", None)
|
||||
|
||||
|
||||
def test_an_event_under_the_consented_level_is_dropped(elevated):
|
||||
_user, client, rows = elevated("DEBUG")
|
||||
r = client.post(
|
||||
"/api/diagnostics/events",
|
||||
json={"events": [_event(level="TRACE", msg="typed: lofi"), _event(level="DEBUG")]},
|
||||
)
|
||||
assert r.json()["stored"] == 1
|
||||
assert [row.level for row in rows()] == ["DEBUG"]
|
||||
|
||||
|
||||
def test_a_timestamp_outside_the_consent_window_is_clamped(elevated):
|
||||
_user, client, rows = elevated("DEBUG")
|
||||
now = datetime.now(timezone.utc)
|
||||
future = (now + timedelta(days=365)).isoformat()
|
||||
past = (now - timedelta(days=30)).isoformat()
|
||||
client.post("/api/diagnostics/events", json={"events": [_event(ts=future), _event(ts=past)]})
|
||||
late, early = rows()
|
||||
assert late.ts <= datetime.now(timezone.utc)
|
||||
assert early.ts >= now - timedelta(hours=diagnostics.ELEVATION_HOURS, minutes=1)
|
||||
|
||||
|
||||
def test_sizes_are_capped(elevated):
|
||||
_user, client, rows = elevated("DEBUG")
|
||||
too_many = [_event() for _ in range(diagnostics_routes.EVENTS_MAX + 1)]
|
||||
assert client.post("/api/diagnostics/events", json={"events": too_many}).status_code == 422
|
||||
|
||||
huge = "x" * (diagnostics_routes.BODY_MAX_BYTES + 1)
|
||||
r = client.post(
|
||||
"/api/diagnostics/events",
|
||||
content=json.dumps({"events": [_event(attrs={"blob": huge})]}),
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
assert r.status_code == 413
|
||||
|
||||
big_attrs = {"blob": "y" * (diagnostics_routes.ATTRS_MAX_BYTES + 10)}
|
||||
assert (
|
||||
client.post("/api/diagnostics/events", json={"events": [_event(attrs=big_attrs)]}).json()[
|
||||
"stored"
|
||||
]
|
||||
== 1
|
||||
)
|
||||
(row,) = rows()
|
||||
assert "blob" not in row.attrs and row.attrs["attrs_dropped_bytes"] > 0
|
||||
|
||||
bad_code = client.post("/api/diagnostics/events", json={"events": [_event(event="Nav View")]})
|
||||
assert bad_code.status_code == 422
|
||||
|
||||
|
||||
def test_the_pace_is_capped_per_user(elevated):
|
||||
_user, client, _rows = elevated("DEBUG")
|
||||
codes = [
|
||||
client.post("/api/diagnostics/events", json={"events": []}).status_code for _ in range(31)
|
||||
]
|
||||
assert codes[:30] == [200] * 30 and codes[30] == 429
|
||||
|
||||
|
||||
def test_what_an_account_stores_per_hour_is_capped_and_said_once(elevated, monkeypatch):
|
||||
user, client, rows = elevated("DEBUG")
|
||||
monkeypatch.setattr(
|
||||
diagnostics_routes, "_stored_bytes", diagnostics_routes.RateLimiter(3_000, 3600)
|
||||
)
|
||||
batch = {"events": [_event(msg="m" * 1_000)]}
|
||||
stored = [client.post("/api/diagnostics/events", json=batch).json()["stored"] for _ in range(4)]
|
||||
assert stored == [1, 1, 0, 0]
|
||||
events = [row.event for row in rows()]
|
||||
# Two batches, then ONE notice for the hour, not one per refused batch.
|
||||
assert events == ["nav.view", "nav.view", "diagnostics.budget_spent"]
|
||||
assert rows()[-1].user_id == user.id
|
||||
|
||||
|
||||
def test_only_a_json_body_is_read(elevated):
|
||||
_user, client, rows = elevated("DEBUG")
|
||||
# What a same-site `<form enctype="text/plain">` can send without a preflight.
|
||||
r = client.post(
|
||||
"/api/diagnostics/events",
|
||||
content=json.dumps({"events": [_event()]}),
|
||||
headers={"content-type": "text/plain"},
|
||||
)
|
||||
assert r.status_code == 415 and r.json()["detail"]["code"] == "json_required"
|
||||
assert rows() == []
|
||||
ok = client.post(
|
||||
"/api/diagnostics/events",
|
||||
content=json.dumps({"events": [_event()]}),
|
||||
headers={"content-type": "application/json; charset=utf-8"},
|
||||
)
|
||||
assert ok.json()["stored"] == 1
|
||||
|
||||
|
||||
# --- the server side of an elevation ---------------------------------------------------------
|
||||
|
||||
|
||||
class _Capture(logging.Handler):
|
||||
"""The stdout handler's filters (context + gate), capturing records."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(level=logging.NOTSET)
|
||||
self.records: list[logging.LogRecord] = []
|
||||
self.addFilter(logs.ContextFilter())
|
||||
self.addFilter(logs.LevelGate())
|
||||
|
||||
def emit(self, record):
|
||||
self.records.append(record)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gated():
|
||||
"""A process configured at INFO with a gated handler on the ROOT (where the real one sits), and
|
||||
every piece of module state the elevation touches restored afterwards."""
|
||||
root = logging.getLogger()
|
||||
saved = (logs._base_level, dict(logs._elevations), logs._root_applied, root.level)
|
||||
handler = _Capture()
|
||||
root.addHandler(handler)
|
||||
logs._elevations.clear()
|
||||
logs.set_level("INFO")
|
||||
try:
|
||||
yield handler
|
||||
finally:
|
||||
root.removeHandler(handler)
|
||||
logs._base_level, logs._root_applied = saved[0], saved[2]
|
||||
logs._elevations.clear()
|
||||
logs._elevations.update(saved[1])
|
||||
root.setLevel(saved[3])
|
||||
|
||||
|
||||
def _app() -> TestClient:
|
||||
"""What a real request does: the request middleware, then a SYNC auth dependency that resolves
|
||||
the account (run by FastAPI in a threadpool, on a copy of the context), then a handler."""
|
||||
app = FastAPI()
|
||||
app.middleware("http")(logs.log_requests)
|
||||
handler_log = logging.getLogger("siftlode.difftest")
|
||||
|
||||
def account(user_id: int, level: str = "INFO"):
|
||||
until = datetime.now(timezone.utc) + timedelta(hours=1)
|
||||
logs.set_user(user_id, level, until)
|
||||
|
||||
@app.get("/api/thing/{user_id}")
|
||||
def _thing(user_id: int, _=Depends(account)):
|
||||
handler_log.debug("detail only an elevated request keeps")
|
||||
handler_log.info("an ordinary line")
|
||||
return {}
|
||||
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_the_account_reaches_the_handlers_own_lines(gated):
|
||||
_app().get("/api/thing/41")
|
||||
line = next(r for r in gated.records if r.getMessage() == "an ordinary line")
|
||||
assert getattr(line, "user_id", None) == 41
|
||||
|
||||
|
||||
def test_one_users_elevation_logs_their_requests_in_detail_and_nobody_elses(gated):
|
||||
client = _app()
|
||||
client.get("/api/thing/7", params={"level": "DEBUG"})
|
||||
client.get("/api/thing/8")
|
||||
detail = [r for r in gated.records if r.getMessage().startswith("detail only")]
|
||||
assert [r.user_id for r in detail] == [7]
|
||||
assert getattr(detail[0], "elevated", False) is True
|
||||
# Every line of the elevated request is marked for storage — the sink keeps it at any level.
|
||||
ordinary = [r for r in gated.records if r.getMessage() == "an ordinary line"]
|
||||
assert [logsink.should_store(r) for r in ordinary] == [True, False]
|
||||
# The process's configured level is still what it reports, however low the root went.
|
||||
assert logs.current_level() == "INFO" and logging.getLogger().level == logging.DEBUG
|
||||
|
||||
|
||||
def test_the_root_level_returns_to_the_base_when_the_elevation_ends(gated):
|
||||
client = _app()
|
||||
client.get("/api/thing/7", params={"level": "TRACE"})
|
||||
assert logging.getLogger().level == logs.TRACE
|
||||
client.get("/api/thing/7") # the same user, back at the base
|
||||
assert logging.getLogger().level == logging.INFO
|
||||
logs._track_elevation(9, logging.DEBUG, datetime.now(timezone.utc) - timedelta(seconds=1))
|
||||
assert logging.getLogger().level == logging.INFO # an already-expired one is pruned at once
|
||||
|
||||
|
||||
def test_the_query_string_is_logged_only_for_a_traced_request_and_redacted(gated):
|
||||
client = _app()
|
||||
client.get("/api/thing/7", params={"level": "DEBUG", "q": "lofi"})
|
||||
client.get("/api/thing/7", params={"level": "TRACE", "q": "lofi", "token": "SECRET"})
|
||||
lines = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||
assert "query" not in lines[0].attrs
|
||||
assert "q=lofi" in lines[1].attrs["query"] and "SECRET" not in lines[1].attrs["query"]
|
||||
|
||||
|
||||
def test_an_address_typed_into_a_search_is_redacted_though_the_browser_encodes_it(gated):
|
||||
_app().get("/api/thing/7?level=TRACE&q=someone%40example.com&new_password=hunter2")
|
||||
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||
assert line.attrs["query"] == "level=TRACE&q=[email]&new_password=[redacted]"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
["state=a%20b", "code=%20SECRET", "sig=a%22b", "key=a%26b", "reset=a+b"],
|
||||
ids=["space", "leading-space", "quote", "ampersand", "plus"],
|
||||
)
|
||||
def test_a_credential_param_goes_whole_however_its_value_decodes(raw):
|
||||
"""Decoding is what lets `_EMAIL_RE` see `%40`, but it also puts spaces and separators into a
|
||||
value that `_SECRET_PARAM_RE`'s value class stops at — so the name rule decides, not the text."""
|
||||
name = raw.split("=", 1)[0]
|
||||
assert logs.redact_query(f"{raw}&q=lofi") == f"{name}={logs.REDACTED}&q=lofi"
|
||||
|
||||
|
||||
def test_an_admin_level_of_trace_does_not_log_other_users_query_strings(gated):
|
||||
logs.set_level("TRACE")
|
||||
_app().get("/api/thing/7", params={"q": "something private"})
|
||||
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||
assert "query" not in line.attrs
|
||||
|
||||
|
||||
def test_the_admin_level_applies_to_the_running_process_when_set_and_when_reset(
|
||||
gated, db, make_user, as_user
|
||||
):
|
||||
admin = as_user(make_user(role="admin"))
|
||||
r = admin.patch("/api/admin/config/log_level", json={"value": "DEBUG"})
|
||||
assert r.status_code == 200, r.text
|
||||
assert logs.current_level() == "DEBUG"
|
||||
assert admin.delete("/api/admin/config/log_level").status_code == 200
|
||||
assert logs.current_level() == settings.log_level
|
||||
|
||||
|
||||
def test_erasing_the_only_elevated_account_puts_the_process_level_back(gated):
|
||||
"""The root level is the lowest of the base and every unexpired elevation. `forget_user` drops
|
||||
an elevation like any other path that drops one, so it owes the same recompute — or the process
|
||||
keeps building every TRACE record for an account that no longer exists, to throw them away."""
|
||||
until = datetime.now(timezone.utc) + timedelta(hours=1)
|
||||
try:
|
||||
logs.set_user(7, "TRACE", until)
|
||||
assert logging.getLogger().level == logs.TRACE
|
||||
logs.forget_user(7)
|
||||
assert logging.getLogger().level == logging.INFO
|
||||
finally:
|
||||
# `set_user` writes the request scope, and outside a request that scope is this process's
|
||||
# for good — a later test would store its INFO lines as an elevated session's.
|
||||
logs.set_user(None)
|
||||
|
||||
|
||||
def test_the_stored_admin_level_is_applied_at_boot(gated, db, monkeypatch):
|
||||
import asyncio
|
||||
|
||||
import app.main as app_main
|
||||
|
||||
sysconfig.set_value(db, "log_level", "DEBUG")
|
||||
db.commit()
|
||||
monkeypatch.setattr(app_main.state, "is_configured", lambda db: True)
|
||||
monkeypatch.setattr(app_main, "start_scheduler", lambda: None)
|
||||
monkeypatch.setattr(app_main, "shutdown_scheduler", lambda: None)
|
||||
monkeypatch.setattr(app_main.plex_subtitles, "shutdown", lambda: None)
|
||||
|
||||
async def _no_sweep():
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(app_main, "_sweep_hls_orphans", _no_sweep)
|
||||
levels: list[str] = []
|
||||
|
||||
async def _cycle():
|
||||
async with app_main.lifespan(app_main.app):
|
||||
levels.append(logs.current_level())
|
||||
|
||||
asyncio.run(_cycle())
|
||||
assert levels == ["DEBUG"]
|
||||
|
||||
|
||||
def test_the_root_level_is_applied_in_the_order_the_elevations_changed(gated, monkeypatch):
|
||||
"""Two requests change the elevations at once: A's user leaves (the root may go back to INFO),
|
||||
B's user arrives (it must go to DEBUG). B's change is the later one, so DEBUG must be what stays
|
||||
— even when A is paused between working out its level and applying it."""
|
||||
root = logging.getLogger()
|
||||
logs._track_elevation(1, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1))
|
||||
real_set = root.setLevel
|
||||
other: list[threading.Thread] = []
|
||||
|
||||
def paused_set(level):
|
||||
if level == logging.INFO and not other:
|
||||
b = threading.Thread(
|
||||
target=logs._track_elevation,
|
||||
args=(2, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1)),
|
||||
)
|
||||
other.append(b)
|
||||
b.start()
|
||||
b.join(0.3) # B runs to completion here unless A's level change holds it off
|
||||
real_set(level)
|
||||
|
||||
monkeypatch.setattr(root, "setLevel", paused_set)
|
||||
logs._track_elevation(1, None, None) # A
|
||||
other[0].join(5)
|
||||
assert root.level == logging.DEBUG and 2 in logs._elevations
|
||||
|
||||
|
||||
def test_a_real_sign_in_carries_the_consented_level_into_the_process(gated, client, db, make_user):
|
||||
user = _signed_in(client, make_user)
|
||||
assert client.put("/api/me/diagnostics", json={"level": "DEBUG"}).status_code == 200
|
||||
logs._elevations.clear() # only a request resolving the account through auth may put it back
|
||||
client.get("/api/me")
|
||||
assert logs._elevations[user.id][0] == logging.DEBUG
|
||||
assert client.put("/api/me/diagnostics", json={"level": "INFO"}).status_code == 200
|
||||
assert user.id not in logs._elevations
|
||||
@@ -0,0 +1,942 @@
|
||||
"""E24 S1 — the Home surface's read-model.
|
||||
|
||||
Covers the four things that can silently rot here: the time-budget bands (pure), the cross-source
|
||||
resume rail's merge order, the hero's tier ladder, and the comfort shelf (its schema-level one-of
|
||||
invariant, its cascade, and the resolved `next` that makes one click enough).
|
||||
|
||||
DB lane (`db` fixture) except for the budget block — skipped without Postgres.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from app.discover import hero, pins, queries, rails
|
||||
from app.discover.budget import BANDS, fits, remaining_seconds
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card
|
||||
from app.models import (
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
ComfortPin,
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
PlexShow,
|
||||
PlexState,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
Subscription,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
|
||||
NOW = datetime(2026, 6, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
# ── budget (pure) ─────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_bands_cover_every_runtime_and_overlap_deliberately():
|
||||
"""Every runtime has a home in `any`, and the bands overlap so "30 perc" is not answered with a
|
||||
two-minute Short just because it fits under the cap."""
|
||||
assert fits(3, "any") and fits(10_000, "any")
|
||||
assert fits(2 * 60, "short") and not fits(2 * 60, "medium") # a Short is not a 30-minute answer
|
||||
assert fits(30 * 60, "medium") and not fits(30 * 60, "short")
|
||||
assert fits(110 * 60, "long") and not fits(110 * 60, "medium")
|
||||
|
||||
|
||||
def test_unknown_duration_only_fits_the_unbounded_band():
|
||||
"""We cannot promise an unknown-length item finishes inside ten minutes, so it is not offered as
|
||||
an answer to a bounded question — the one control on the page has to mean what it says."""
|
||||
for budget in BANDS:
|
||||
assert fits(None, budget) is (budget == "any")
|
||||
|
||||
|
||||
def test_budget_is_matched_against_what_is_LEFT():
|
||||
"""A 90-minute film with 8 minutes to go is a ten-minute answer."""
|
||||
assert remaining_seconds(90 * 60, 82 * 60) == 8 * 60
|
||||
assert fits(remaining_seconds(90 * 60, 82 * 60), "short")
|
||||
assert not fits(remaining_seconds(90 * 60, 0), "short")
|
||||
assert remaining_seconds(None, 30) is None
|
||||
assert remaining_seconds(600, 900) == 0 # a position past the end never goes negative
|
||||
|
||||
|
||||
# ── fixtures ──────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _seed_yt(db) -> tuple[User, Channel]:
|
||||
"""A subscribed channel with three videos of distinct lengths, so a budget can pick between them."""
|
||||
user = User(email="discover-e24@example.com")
|
||||
ch = Channel(id="UCe24", title="Comfort Channel", handle="@comfort")
|
||||
db.add_all([user, ch])
|
||||
db.flush()
|
||||
db.add(Subscription(user_id=user.id, channel_id=ch.id))
|
||||
db.add_all(
|
||||
[
|
||||
Video(
|
||||
id="vshort",
|
||||
channel_id=ch.id,
|
||||
title="Ten minute video",
|
||||
duration_seconds=9 * 60,
|
||||
published_at=NOW - timedelta(days=1),
|
||||
),
|
||||
Video(
|
||||
id="vmed",
|
||||
channel_id=ch.id,
|
||||
title="Half hour video",
|
||||
duration_seconds=30 * 60,
|
||||
published_at=NOW - timedelta(days=2),
|
||||
),
|
||||
Video(
|
||||
id="vlong",
|
||||
channel_id=ch.id,
|
||||
title="Feature length video",
|
||||
duration_seconds=95 * 60,
|
||||
published_at=NOW - timedelta(days=3),
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
return user, ch
|
||||
|
||||
|
||||
def _seed_plex(db, user: User) -> tuple[PlexItem, PlexShow, PlexItem]:
|
||||
lib_m = PlexLibrary(plex_key="90", title="Movies", kind="movie", enabled=True)
|
||||
lib_s = PlexLibrary(plex_key="91", title="Shows", kind="show", enabled=True)
|
||||
db.add_all([lib_m, lib_s])
|
||||
db.flush()
|
||||
movie = PlexItem(
|
||||
rating_key="pm1",
|
||||
library_id=lib_m.id,
|
||||
kind="movie",
|
||||
title="A Movie",
|
||||
duration_s=100 * 60,
|
||||
playable="direct",
|
||||
)
|
||||
show = PlexShow(rating_key="ps1", library_id=lib_s.id, title="A Show", child_count=1)
|
||||
db.add_all([movie, show])
|
||||
db.flush()
|
||||
ep1 = PlexItem(
|
||||
rating_key="ps1e1",
|
||||
library_id=lib_s.id,
|
||||
kind="episode",
|
||||
title="Ep 1",
|
||||
show_id=show.id,
|
||||
season_number=1,
|
||||
episode_number=1,
|
||||
duration_s=45 * 60,
|
||||
)
|
||||
ep2 = PlexItem(
|
||||
rating_key="ps1e2",
|
||||
library_id=lib_s.id,
|
||||
kind="episode",
|
||||
title="Ep 2",
|
||||
show_id=show.id,
|
||||
season_number=1,
|
||||
episode_number=2,
|
||||
duration_s=45 * 60,
|
||||
)
|
||||
db.add_all([ep1, ep2])
|
||||
db.commit()
|
||||
return movie, show, ep1
|
||||
|
||||
|
||||
def _enable_plex(db):
|
||||
from app import sysconfig
|
||||
|
||||
sysconfig.set_value(db, "plex_enabled", True)
|
||||
db.commit()
|
||||
|
||||
|
||||
# ── continue watching ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_continue_watching_merges_both_sources_by_recency(db):
|
||||
"""One rail, not two lists side by side — and ordered by when the user last touched each thing,
|
||||
across sources. Two side-by-side rails would just move the decision up a level."""
|
||||
user, _ = _seed_yt(db)
|
||||
movie, _, _ = _seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
db.add_all(
|
||||
[
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vmed",
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW - timedelta(hours=5),
|
||||
),
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=movie.id,
|
||||
status="new",
|
||||
position_seconds=600,
|
||||
progress_updated_at=NOW - timedelta(hours=1),
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
got = rails.continue_watching(db, user)
|
||||
assert [(c["source"], c["id"]) for c in got] == [("plex", "pm1"), ("youtube", "vmed")]
|
||||
assert got[0]["remaining_seconds"] == 100 * 60 - 600
|
||||
|
||||
|
||||
def test_continue_watching_is_youtube_only_when_plex_is_off(db):
|
||||
"""Plex is a deployment toggle; with it off the surface says nothing about Plex at all."""
|
||||
user, _ = _seed_yt(db)
|
||||
movie, _, _ = _seed_plex(db, user)
|
||||
db.add_all(
|
||||
[
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vmed",
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW,
|
||||
),
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=movie.id,
|
||||
status="new",
|
||||
position_seconds=600,
|
||||
progress_updated_at=NOW,
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
assert {c["source"] for c in rails.continue_watching(db, user)} == {"youtube"}
|
||||
|
||||
|
||||
def test_the_resume_rail_obeys_the_channel_blocklist(db):
|
||||
"""Blocking a channel does not touch `video_states`, so a half-watched video from it survives
|
||||
with its resume position — and this rail builds its own SELECT instead of going through
|
||||
`filtered_query`. "Never show me this channel" has to outrank "you started it", or the landing
|
||||
page answers "what should I watch" with the thing the user has just banned."""
|
||||
user, ch = _seed_yt(db)
|
||||
db.add(
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vmed",
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
assert [c["id"] for c in rails.continue_watching(db, user)] == ["vmed"]
|
||||
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=ch.id))
|
||||
db.commit()
|
||||
assert rails.continue_watching(db, user) == []
|
||||
# And so the hero, whose resume tier draws from that rail, has nothing to offer either.
|
||||
assert hero.pick(db, user, "any", 0) is None
|
||||
|
||||
|
||||
def test_a_null_timestamp_sorts_last(db):
|
||||
"""A missing progress timestamp sorts last rather than 500ing the landing page.
|
||||
|
||||
Narrower than it was written to be, and the epic-end review corrected it: the previous key,
|
||||
`(ts is not None, ts)`, does NOT raise on two nulls — tuple comparison finds the first UNEQUAL
|
||||
element via `==`, so `None < None` is never reached, and this test passes under that key too.
|
||||
What it does discriminate is the naive `key=lambda r: r[0]`, which really does raise. See
|
||||
`plex.rollup.NO_TS` for the corrected rationale."""
|
||||
user, _ = _seed_yt(db)
|
||||
db.add_all(
|
||||
[
|
||||
VideoState(
|
||||
user_id=user.id, video_id="vmed", status="new", position_seconds=1
|
||||
), # no progress_updated_at
|
||||
VideoState(
|
||||
user_id=user.id, video_id="vlong", status="new", position_seconds=1
|
||||
), # nor here
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vshort",
|
||||
status="new",
|
||||
position_seconds=1,
|
||||
progress_updated_at=NOW,
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
got = rails.continue_watching(db, user)
|
||||
assert got[0]["id"] == "vshort"
|
||||
assert len(got) == 3
|
||||
|
||||
|
||||
# ── hero ──────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_hero_prefers_resume_and_labels_the_tier(db):
|
||||
user, _ = _seed_yt(db)
|
||||
db.add(
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vmed",
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
got = hero.pick(db, user, "any", seed=1)
|
||||
assert got["tier"] == "resume"
|
||||
assert got["card"]["id"] == "vmed"
|
||||
|
||||
|
||||
def test_hero_skips_a_resume_that_does_not_fit_the_budget(db):
|
||||
""" "I have ten minutes" must not be answered with the 80 minutes left on a film, even though
|
||||
that film is the most recent thing the user touched."""
|
||||
user, _ = _seed_yt(db)
|
||||
db.add(
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vlong",
|
||||
status="new",
|
||||
position_seconds=60,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
got = hero.pick(db, user, "short", seed=1)
|
||||
assert got["tier"] == "surprise"
|
||||
assert got["card"]["duration_seconds"] <= BANDS["short"][1]
|
||||
|
||||
|
||||
def test_hero_falls_back_to_the_shelf_before_guessing(db):
|
||||
"""The user's own declared fallback beats anything we could infer."""
|
||||
user, ch = _seed_yt(db)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
||||
db.commit()
|
||||
got = hero.pick(db, user, "medium", seed=1)
|
||||
assert got["tier"] == "comfort"
|
||||
assert got["card"]["id"] == "vmed"
|
||||
|
||||
|
||||
def test_hero_resolves_a_show_to_the_episode_that_would_play(db):
|
||||
"""A show has no Play button. The hero of a partially-watched show is its next episode."""
|
||||
user, _ = _seed_yt(db)
|
||||
_, show, ep1 = _seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=ep1.id,
|
||||
status="watched",
|
||||
watched_at=NOW,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
got = hero.pick(db, user, "any", seed=1)
|
||||
assert got["tier"] == "resume"
|
||||
assert got["card"]["kind"] == "episode"
|
||||
assert got["card"]["id"] == "ps1e2" # ep1 is finished → ep2 is what plays
|
||||
|
||||
|
||||
def test_hero_is_none_when_nothing_fits(db):
|
||||
"""An empty answer is an honest answer: the page renders its own empty state rather than being
|
||||
handed a card that does not answer what was asked. Here the whole library is under 40 minutes,
|
||||
so "egy este" has nothing to offer."""
|
||||
user, ch = _seed_yt(db)
|
||||
db.execute(text("DELETE FROM videos WHERE id = 'vlong'"))
|
||||
db.commit()
|
||||
assert hero.pick(db, user, "long", seed=1) is None
|
||||
|
||||
|
||||
def test_surprise_is_reproducible_for_a_seed(db):
|
||||
"""The reroll button depends on this: one seed drives both sources, so the same seed must give
|
||||
the same answer and a different seed must be free to give another."""
|
||||
user, _ = _seed_yt(db)
|
||||
a = rails.surprise(db, user, "any", seed=7, limit=3)
|
||||
b = rails.surprise(db, user, "any", seed=7, limit=3)
|
||||
assert [c["id"] for c in a] == [c["id"] for c in b]
|
||||
seeds = {tuple(c["id"] for c in rails.surprise(db, user, "any", s, 3)) for s in range(12)}
|
||||
assert len(seeds) > 1 # the seed actually reaches the ordering
|
||||
|
||||
|
||||
def test_surprise_never_offers_something_already_started(db):
|
||||
"""The resume tier may have skipped an item because its REMAINING time did not fit; handing the
|
||||
same item back as "Meglepetés" — where the full runtime does fit — would contradict the question
|
||||
the user just answered."""
|
||||
user, _ = _seed_yt(db)
|
||||
db.add(
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vlong",
|
||||
status="new",
|
||||
position_seconds=90 * 60,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
assert "vlong" not in {c["id"] for c in rails.surprise(db, user, "long", seed=3, limit=5)}
|
||||
|
||||
|
||||
def test_the_seed_decides_which_source_leads(db):
|
||||
"""With a pool of one, a fixed source order would mean the Plex library could never answer "egy
|
||||
este" whenever a YouTube video also fit — the cross-source promise would quietly be a YouTube
|
||||
promise."""
|
||||
user, _ = _seed_yt(db)
|
||||
_seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
leaders = {rails.surprise(db, user, "any", s, limit=1)[0]["source"] for s in range(6)}
|
||||
assert leaders == {"youtube", "plex"}
|
||||
|
||||
|
||||
def test_the_reroll_must_be_able_to_move_past_the_resume_tier(db):
|
||||
"""A new seed alone only re-draws the SURPRISE tier — the resume tier deterministically returns
|
||||
the most recently progressed thing that fits, so without an exclusion the reroll handed the same
|
||||
card straight back and the button looked dead (found in the browser, 2026-09-06)."""
|
||||
user, _ = _seed_yt(db)
|
||||
db.add_all(
|
||||
[
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vmed",
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW,
|
||||
),
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vshort",
|
||||
status="new",
|
||||
position_seconds=60,
|
||||
progress_updated_at=NOW - timedelta(hours=1),
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
first = hero.pick(db, user, "any", seed=1)
|
||||
assert first["card"]["id"] == "vmed"
|
||||
# Same seed, same everything — only the exclusion differs.
|
||||
second = hero.pick(db, user, "any", seed=1, exclude={"youtube:vmed"})
|
||||
assert second["card"]["id"] == "vshort"
|
||||
|
||||
|
||||
def test_the_reroll_rotates_which_tier_answers(db):
|
||||
"""Measured 2026-09-06: exclusion alone left the button walking down the resume rail (20 rows)
|
||||
and cycling nine cards, every one of them `resume` — the shelf and the surprise tier were
|
||||
unreachable by pressing it. `start` is the press count, so consecutive presses offer a different
|
||||
KIND of answer."""
|
||||
user, _ = _seed_yt(db)
|
||||
# One of each: something half-finished, something pinned, and a third video for the draw.
|
||||
db.add(
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vmed",
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vlong", position=0))
|
||||
db.commit()
|
||||
tiers = [hero.pick(db, user, "any", seed=1, start=n)["tier"] for n in range(4)]
|
||||
assert tiers == ["resume", "comfort", "surprise", "resume"]
|
||||
|
||||
|
||||
def test_the_rotation_still_falls_through_an_empty_tier(db):
|
||||
"""A press must not come back empty just because the tier it happened to lead with has nothing.
|
||||
Here there is no shelf at all, so the comfort-led press has to answer from somewhere else."""
|
||||
user, _ = _seed_yt(db)
|
||||
assert db.execute(select(func.count()).select_from(ComfortPin)).scalar_one() == 0
|
||||
got = hero.pick(db, user, "any", seed=1, start=1)
|
||||
assert got is not None and got["tier"] != "comfort"
|
||||
|
||||
|
||||
def test_excluding_a_show_also_excludes_the_episode_it_resolves_to(db):
|
||||
"""The user rerolled past the SHOW they were shown; handing back its next episode under a
|
||||
different title would be the same suggestion wearing a hat."""
|
||||
user, _ = _seed_yt(db)
|
||||
_, show, ep1 = _seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=ep1.id,
|
||||
status="new",
|
||||
position_seconds=120,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
shown = hero.pick(db, user, "any", seed=1)
|
||||
assert shown["card"]["id"] == "ps1e1" # the show resolved to its in-progress episode
|
||||
# Excluding what the USER saw — the show — must move the tier on, not return the episode again.
|
||||
after = hero.pick(db, user, "any", seed=1, exclude={"plex:ps1"})
|
||||
assert after is None or after["card"]["id"] != "ps1e1"
|
||||
|
||||
|
||||
# ── comfort shelf ─────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_a_pin_must_reference_exactly_one_thing(db):
|
||||
"""The union invariant lives in the schema, not in a comment at every insert site."""
|
||||
user, ch = _seed_yt(db)
|
||||
for kwargs in ({}, {"video_id": "vmed", "channel_id": ch.id}):
|
||||
db.add(ComfortPin(user_id=user.id, position=0, **kwargs))
|
||||
with pytest.raises(IntegrityError):
|
||||
db.flush()
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_the_same_thing_cannot_be_pinned_twice(db):
|
||||
"""The per-kind unique constraints are what make the route's concurrent-tab path safe (it catches
|
||||
`IntegrityError` precisely because of them) — and the migration drift check filters constraint
|
||||
diffs out as noise, so nothing else would notice them going missing. The route's own dedupe
|
||||
returns early without ever reaching the DB, so only this test exercises them."""
|
||||
user, _ = _seed_yt(db)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
||||
db.commit()
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=1))
|
||||
with pytest.raises(IntegrityError):
|
||||
db.flush()
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_deleting_the_pinned_row_removes_the_pin(db):
|
||||
"""A film dropped from the library must not leave a dead tile on the one surface whose whole job
|
||||
is to be reliable — which is why the shelf holds foreign keys, not strings."""
|
||||
user, _ = _seed_yt(db)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
||||
db.commit()
|
||||
db.execute(text("DELETE FROM videos WHERE id = 'vmed'"))
|
||||
db.commit()
|
||||
assert db.execute(select(func.count()).select_from(ComfortPin)).scalar() == 0
|
||||
|
||||
|
||||
def test_shelf_resolves_a_channel_to_its_newest_unwatched_video(db):
|
||||
""" "Put that channel on" has to be one click, so the tile carries what would play."""
|
||||
user, ch = _seed_yt(db)
|
||||
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=0))
|
||||
db.commit()
|
||||
(card,) = pins.shelf(db, user)
|
||||
assert card["kind"] == "channel"
|
||||
assert card["unwatched_count"] == 3
|
||||
assert card["next"]["id"] == "vshort" # newest
|
||||
|
||||
|
||||
def test_shelf_resolves_a_playlist_to_its_first_unwatched_item(db):
|
||||
user, _ = _seed_yt(db)
|
||||
pl = Playlist(user_id=user.id, name="Elalvós")
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
db.add_all(
|
||||
[
|
||||
PlaylistItem(playlist_id=pl.id, video_id="vshort", position=0),
|
||||
PlaylistItem(playlist_id=pl.id, video_id="vmed", position=1),
|
||||
]
|
||||
)
|
||||
db.add(VideoState(user_id=user.id, video_id="vshort", status="watched", watched_at=NOW))
|
||||
db.add(ComfortPin(user_id=user.id, playlist_id=pl.id, position=0))
|
||||
db.commit()
|
||||
(card,) = pins.shelf(db, user)
|
||||
assert card["kind"] == "playlist"
|
||||
assert card["item_count"] == 2
|
||||
assert card["next"]["id"] == "vmed"
|
||||
|
||||
|
||||
def test_a_pinned_video_survives_leaving_the_feed_scope(db):
|
||||
"""The shelf is the fallback surface. An entry silently vanishing because the user unsubscribed
|
||||
from the channel is exactly the failure it must not have."""
|
||||
user, ch = _seed_yt(db)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
||||
db.commit()
|
||||
db.execute(delete(Subscription))
|
||||
db.commit()
|
||||
(card,) = pins.shelf(db, user)
|
||||
assert card["id"] == "vmed"
|
||||
assert card["next"]["id"] == "vmed"
|
||||
|
||||
|
||||
def test_the_plex_surprise_obeys_the_time_budget(db):
|
||||
"""The one control on the page has to mean what it says: a 100-minute film is not an answer to
|
||||
"10 perc". The YouTube half of this rule has had its own test since S1; the Plex half's duration
|
||||
bounds could be deleted with the whole suite still green, because every test that reached
|
||||
`plex_surprise` passed budget="any"."""
|
||||
user, _ = _seed_yt(db)
|
||||
_seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
drawn = lambda budget: { # noqa: E731
|
||||
(c["source"], c["id"]) for s in range(6) for c in rails.surprise(db, user, budget, s, 5)
|
||||
}
|
||||
assert ("plex", "pm1") not in drawn("short")
|
||||
# …and it IS offered where it fits, so this cannot pass by the rail being empty.
|
||||
assert ("plex", "pm1") in drawn("long")
|
||||
|
||||
|
||||
def test_the_plex_surprise_never_offers_something_already_started(db):
|
||||
"""A half-watched film is not a surprise, and the resume tier has already had its chance at it.
|
||||
The mirror of `test_surprise_never_offers_something_already_started` for the other source — the
|
||||
asymmetric half of one principle, which the house rules say to fix everywhere."""
|
||||
user, _ = _seed_yt(db)
|
||||
movie, _, _ = _seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=movie.id,
|
||||
status="new",
|
||||
position_seconds=600,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
drawn = {
|
||||
(c["source"], c["id"]) for s in range(6) for c in rails.surprise(db, user, "long", s, 5)
|
||||
}
|
||||
assert ("plex", "pm1") not in drawn
|
||||
|
||||
|
||||
def test_a_new_pin_lands_at_the_end_even_after_a_hole(db):
|
||||
"""Positions are never renumbered on removal — `remove_pin` deletes the row, and every FK cascade
|
||||
silently removes a pin when its target disappears — so the append position has to come from
|
||||
`max(position)`, not from the row count. With a count, the first hole drops the next pin into the
|
||||
MIDDLE of the shelf and the second makes two pins claim one slot."""
|
||||
user, ch = _seed_yt(db)
|
||||
db.add_all(
|
||||
[
|
||||
ComfortPin(user_id=user.id, video_id="vshort", position=0),
|
||||
ComfortPin(user_id=user.id, video_id="vmed", position=1),
|
||||
ComfortPin(user_id=user.id, video_id="vlong", position=2),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
db.execute(delete(ComfortPin).where(ComfortPin.video_id.in_(("vshort", "vmed"))))
|
||||
db.commit()
|
||||
|
||||
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=pins.next_position(db, user)))
|
||||
db.commit()
|
||||
assert [c["kind"] for c in pins.shelf(db, user)] == ["video", "channel"]
|
||||
positions = (
|
||||
db.execute(select(ComfortPin.position).order_by(ComfortPin.position)).scalars().all()
|
||||
)
|
||||
assert len(set(positions)) == len(positions) # and no two tiles claim the same slot
|
||||
|
||||
|
||||
def test_a_self_playing_pin_does_not_reference_itself(db):
|
||||
"""`next` must be a distinct dict even when the pin plays itself. Aliasing the card into its own
|
||||
`next` is a cycle, and the serializer turns the whole shelf into a 500 over it."""
|
||||
user, _ = _seed_yt(db)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
||||
db.commit()
|
||||
(card,) = pins.shelf(db, user)
|
||||
assert card["next"] is not card
|
||||
assert "next" not in card["next"]
|
||||
|
||||
|
||||
def test_the_resume_rail_says_what_would_actually_play(db):
|
||||
"""A show tile that only opens the show page moves the decision instead of removing it — the
|
||||
click lands on six seasons to choose between, which is the paralysis this whole surface exists
|
||||
for. The shelf and the hero already resolved; this rail did not (reported 2026-09-06). The tile
|
||||
stays the show (its artwork is what the user recognises); `next` is what starts."""
|
||||
user, _ = _seed_yt(db)
|
||||
_, show, ep1 = _seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=ep1.id,
|
||||
status="watched",
|
||||
watched_at=NOW,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
rail = rails.continue_watching(db, user)
|
||||
tile = next(c for c in rail if c["kind"] == "show")
|
||||
assert tile["id"] == "ps1" # still the show
|
||||
assert tile["next"]["kind"] == "episode"
|
||||
assert tile["next"]["id"] == "ps1e2" # ep1 is finished → ep2 is what plays
|
||||
|
||||
|
||||
def test_only_a_show_row_pays_for_a_resolution(db):
|
||||
"""`next` is the answer to "a show has no Play button". Attaching it to a movie or a video would
|
||||
both cost a query for nothing and re-open the self-reference cycle the shelf already hit.
|
||||
|
||||
The states are what make this test exist: without them every rail query inner-joins its way to
|
||||
zero rows, the loop body never runs, and the assertion is vacuous — which is exactly how the
|
||||
epic-end review found it."""
|
||||
user, _ = _seed_yt(db)
|
||||
movie, _, ep1 = _seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
db.add_all(
|
||||
[
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vmed",
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW,
|
||||
),
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=movie.id,
|
||||
status="new",
|
||||
position_seconds=600,
|
||||
progress_updated_at=NOW,
|
||||
),
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=ep1.id,
|
||||
status="new",
|
||||
position_seconds=120,
|
||||
progress_updated_at=NOW,
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
rail = rails.continue_watching(db, user)
|
||||
# A video, a movie and a show — one row of each kind that can appear here, so the assertion below
|
||||
# actually exercises both sides of the equality.
|
||||
assert {c["kind"] for c in rail} == {"video", "movie", "show"}
|
||||
for card in rail:
|
||||
assert ("next" in card) == (card["kind"] == "show"), card["id"]
|
||||
|
||||
|
||||
def test_next_episode_takes_an_episode_with_no_state_row_at_all(db):
|
||||
"""The NULL trap in the SQL rewrite: `status NOT IN ('watched','hidden')` is UNKNOWN — not true —
|
||||
for an episode the user has never touched, so a bare filter drops exactly the common case and the
|
||||
rail would offer episode 1 of a show already half-watched."""
|
||||
user, _ = _seed_yt(db)
|
||||
_, show, ep1 = _seed_plex(db, user)
|
||||
db.add(PlexState(user_id=user.id, item_id=ep1.id, status="watched", watched_at=NOW))
|
||||
db.commit() # ep2 has NO PlexState row
|
||||
assert queries.next_episode(db, user.id, show.id)["id"] == "ps1e2"
|
||||
|
||||
|
||||
def test_a_finished_show_starts_again_from_the_top(db):
|
||||
"""Nothing unwatched left is not "nothing to play" — it is "watch it again"."""
|
||||
user, _ = _seed_yt(db)
|
||||
_, show, ep1 = _seed_plex(db, user)
|
||||
ep2 = db.execute(select(PlexItem).where(PlexItem.rating_key == "ps1e2")).scalar_one()
|
||||
db.add_all(
|
||||
[
|
||||
PlexState(user_id=user.id, item_id=ep1.id, status="watched", watched_at=NOW),
|
||||
PlexState(user_id=user.id, item_id=ep2.id, status="watched", watched_at=NOW),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
assert queries.next_episode(db, user.id, show.id)["id"] == "ps1e1"
|
||||
|
||||
|
||||
def test_next_episode_agrees_with_the_show_page_when_two_are_in_progress(db):
|
||||
"""One show, one answer. `plex.rollup.rollup` — which drives the show page's Resume button —
|
||||
takes the LAST in-progress episode in broadcast order; taking the FIRST here gave the same show
|
||||
two different "continue" answers, with the rail ranking it by its most recent touch and then
|
||||
offering the episode abandoned weeks earlier. Two in-progress episodes are not exotic: a scrub to
|
||||
the end leaves a stale `status='new', position>0` row until the daily reconcile."""
|
||||
user, _ = _seed_yt(db)
|
||||
_, show, ep1 = _seed_plex(db, user)
|
||||
ep2 = db.execute(select(PlexItem).where(PlexItem.rating_key == "ps1e2")).scalar_one()
|
||||
db.add_all(
|
||||
[
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=ep1.id,
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW - timedelta(days=9),
|
||||
),
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=ep2.id,
|
||||
status="new",
|
||||
position_seconds=600,
|
||||
progress_updated_at=NOW,
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
got = queries.next_episode(db, user.id, show.id)
|
||||
assert got["id"] == "ps1e2"
|
||||
# Asserted against the other implementation rather than against a literal, so the two cannot
|
||||
# drift apart again without this failing.
|
||||
ordered = [
|
||||
leaf_card(ep, db.execute(select(PlexState).where(PlexState.item_id == ep.id)).scalar_one())
|
||||
for ep in (ep1, ep2)
|
||||
]
|
||||
assert rollup.rollup(ordered)["resume"]["id"] == got["id"]
|
||||
|
||||
|
||||
def test_next_episode_prefers_the_one_in_progress(db):
|
||||
"""In progress beats "next unwatched": resuming is cheaper than starting."""
|
||||
user, _ = _seed_yt(db)
|
||||
_, show, ep1 = _seed_plex(db, user)
|
||||
ep2 = db.execute(select(PlexItem).where(PlexItem.rating_key == "ps1e2")).scalar_one()
|
||||
db.add_all(
|
||||
[
|
||||
PlexState(user_id=user.id, item_id=ep1.id, status="watched", watched_at=NOW),
|
||||
PlexState(user_id=user.id, item_id=ep2.id, status="new", position_seconds=120),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
assert queries.next_episode(db, user.id, show.id)["id"] == "ps1e2"
|
||||
|
||||
|
||||
# ── HTTP surface ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_discover_endpoint_answers_in_one_pass(db, as_user):
|
||||
user, _ = _seed_yt(db)
|
||||
db.add(
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vmed",
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
r = as_user(user).get("/api/discover?budget=medium")
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["budget"] == "medium"
|
||||
assert body["hero"]["tier"] == "resume"
|
||||
assert [c["id"] for c in body["continue_watching"]] == ["vmed"]
|
||||
assert body["comfort"] == []
|
||||
|
||||
|
||||
def test_an_unknown_budget_is_refused_rather_than_silently_widened(db, as_user):
|
||||
user, _ = _seed_yt(db)
|
||||
assert as_user(user).get("/api/discover?budget=whenever").status_code == 422
|
||||
|
||||
|
||||
def test_pinning_twice_is_the_same_shelf(db, as_user):
|
||||
"""The gesture is "make sure this is on my shelf", so a re-pin is a no-op, not a 409."""
|
||||
user, _ = _seed_yt(db)
|
||||
c = as_user(user)
|
||||
first = c.post("/api/discover/pins", json={"kind": "video", "ref": "vmed"}).json()
|
||||
second = c.post("/api/discover/pins", json={"kind": "video", "ref": "vmed"}).json()
|
||||
assert first["created"] is True and second["created"] is False
|
||||
assert first["id"] == second["id"]
|
||||
assert len(c.get("/api/discover/pins").json()["items"]) == 1
|
||||
|
||||
|
||||
def test_pinning_a_plex_show_resolves_to_the_episode_that_would_play(db, as_user):
|
||||
"""Two of the five pinnable kinds are Plex, and the rating_key → internal-id translation is
|
||||
theirs alone — a wrong column or a wrong model in either arm pins the wrong row, or 404s every
|
||||
Plex pin. Neither arm had a test before the epic-end review."""
|
||||
user, _ = _seed_yt(db)
|
||||
_, _, ep1 = _seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
db.add(PlexState(user_id=user.id, item_id=ep1.id, status="watched", watched_at=NOW))
|
||||
db.commit()
|
||||
c = as_user(user)
|
||||
assert c.post("/api/discover/pins", json={"kind": "plex_show", "ref": "ps1"}).status_code == 200
|
||||
(tile,) = c.get("/api/discover/pins").json()["items"]
|
||||
assert (tile["kind"], tile["id"]) == ("show", "ps1")
|
||||
assert tile["next"]["id"] == "ps1e2" # ep1 is finished
|
||||
|
||||
|
||||
def test_pinning_a_plex_movie_plays_itself_without_a_cycle(db, as_user):
|
||||
"""The self-reference that once 500'd the whole shelf, on the other leaf kind: `next` has to be a
|
||||
distinct dict, not the tile aliased into itself. Covered for a video since S1, never for a movie."""
|
||||
user, _ = _seed_yt(db)
|
||||
_seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
c = as_user(user)
|
||||
assert c.post("/api/discover/pins", json={"kind": "plex_item", "ref": "pm1"}).status_code == 200
|
||||
(tile,) = c.get("/api/discover/pins").json()["items"]
|
||||
assert tile["id"] == "pm1" and tile["next"]["id"] == "pm1"
|
||||
|
||||
|
||||
def test_the_reroll_endpoint_parses_its_whole_exclusion_list_and_its_press_count(db, as_user):
|
||||
"""The reroll's behaviour is covered at the domain level; the ROUTE's query-string translation —
|
||||
the comma split and the `start` passthrough — was not covered at all, so a change there could
|
||||
hand back the same card on every press with the suite green. Excluding all three candidates is
|
||||
what proves the whole list parsed: a delimiter change leaves one unsplit string in the set, and a
|
||||
hero comes back.
|
||||
|
||||
NOT the `MAX_EXCLUDE` clamp, which this sends three keys against a cap of 32 and cannot see. The
|
||||
docstring used to claim it did (round 2). The clamp is deliberately left unpinned: it guards
|
||||
nothing the route would otherwise do — see the constant's own comment — and a test that sent 33
|
||||
keys could only assert that the request still answers, which the comma split already proves."""
|
||||
user, _ = _seed_yt(db)
|
||||
db.add(
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id="vmed",
|
||||
status="new",
|
||||
position_seconds=300,
|
||||
progress_updated_at=NOW,
|
||||
)
|
||||
)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vlong", position=0))
|
||||
db.commit()
|
||||
c = as_user(user)
|
||||
|
||||
# `start` rotates which tier answers first: 0 leads with resume, 1 with the shelf.
|
||||
assert c.get("/api/discover/pick?seed=1&start=0").json()["hero"]["tier"] == "resume"
|
||||
assert c.get("/api/discover/pick?seed=1&start=1").json()["hero"]["tier"] == "comfort"
|
||||
|
||||
skip = ",".join(f"youtube:{v}" for v in ("vshort", "vmed", "vlong"))
|
||||
assert c.get(f"/api/discover/pick?seed=2&exclude={skip}").json()["hero"] is None
|
||||
assert c.get("/api/discover/pick?budget=whenever").status_code == 422
|
||||
|
||||
|
||||
def test_pinning_something_that_does_not_exist_is_a_404(db, as_user):
|
||||
user, _ = _seed_yt(db)
|
||||
r = as_user(user).post("/api/discover/pins", json={"kind": "video", "ref": "nope"})
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
def test_a_pin_whose_target_vanishes_mid_request_is_a_404_not_a_500(db, as_user, monkeypatch):
|
||||
"""`add_pin`'s `except IntegrityError` arm was reached by no test at all, so the 500→404
|
||||
narrowing could be reverted with the whole suite green (round 2). The 404 above comes from the
|
||||
`resolve_ref` PRE-CHECK, a different branch that returns before the INSERT.
|
||||
|
||||
The race the arm exists for — a Plex sync dropping the row between `resolve_ref` confirming it
|
||||
and the INSERT — needs two transactions, but the BRANCH does not: a `resolve_ref` pointing at an
|
||||
id no row carries lets the pre-check pass and trips the same FK on commit, with the re-query
|
||||
finding no winner. The sentinel has to FIT `comfort_pins.video_id` (varchar(16)) — a longer one
|
||||
raises `DataError` on the way in, which this arm does not catch and never meant to. Its sibling arm (a concurrent duplicate, where the re-query DOES find a winner
|
||||
and answers 200) genuinely needs a second connection and stays uncovered."""
|
||||
user, _ = _seed_yt(db)
|
||||
monkeypatch.setattr(pins, "resolve_ref", lambda *a, **k: ("video_id", "gone-mid-req"))
|
||||
r = as_user(user).post("/api/discover/pins", json={"kind": "video", "ref": "vshort"})
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
def test_another_users_playlist_is_not_pinnable(db, as_user):
|
||||
"""404, not 403 — a 403 would confirm the id exists to a stranger."""
|
||||
user, _ = _seed_yt(db)
|
||||
other = User(email="other-e24@example.com")
|
||||
db.add(other)
|
||||
db.flush()
|
||||
pl = Playlist(user_id=other.id, name="Theirs")
|
||||
db.add(pl)
|
||||
db.commit()
|
||||
r = as_user(user).post("/api/discover/pins", json={"kind": "playlist", "ref": str(pl.id)})
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
def test_reorder_ignores_ids_that_are_not_yours(db, as_user):
|
||||
"""A stale tab replaying an order that includes a since-deleted pin should reorder what still
|
||||
exists rather than fail wholesale."""
|
||||
user, ch = _seed_yt(db)
|
||||
c = as_user(user)
|
||||
a = c.post("/api/discover/pins", json={"kind": "video", "ref": "vmed"}).json()["id"]
|
||||
b = c.post("/api/discover/pins", json={"kind": "channel", "ref": ch.id}).json()["id"]
|
||||
assert c.post("/api/discover/pins/reorder", json={"ids": [b, 9999, a]}).status_code == 200
|
||||
assert [p["pin_id"] for p in c.get("/api/discover/pins").json()["items"]] == [b, a]
|
||||
|
||||
|
||||
def test_deleting_a_pin_reports_what_went(db, as_user):
|
||||
user, _ = _seed_yt(db)
|
||||
c = as_user(user)
|
||||
pin_id = c.post("/api/discover/pins", json={"kind": "video", "ref": "vmed"}).json()["id"]
|
||||
r = c.delete(f"/api/discover/pins/{pin_id}")
|
||||
assert r.status_code == 200 and r.json() == {"deleted": pin_id}
|
||||
assert c.get("/api/discover/pins").json()["items"] == []
|
||||
@@ -18,7 +18,7 @@ from cryptography.fernet import Fernet
|
||||
|
||||
from app import security
|
||||
from app.downloads import credentials, formats, service
|
||||
from app.models import DownloadCredential, MediaAsset, User
|
||||
from app.models import DownloadCredential, DownloadJob, MediaAsset, User
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -39,13 +39,23 @@ class TestHostKeyForUrl:
|
||||
assert credentials.host_key_for_url("https://www.facebook.com/watch?v=1") == "facebook"
|
||||
assert credentials.host_key_for_url("https://fb.watch/abc/") == "facebook"
|
||||
assert credentials.host_key_for_url("https://x.com/i/status/1") == "twitter"
|
||||
# YouTube gates individual videos behind its bot check; the cookie is the only way past it.
|
||||
for url in (
|
||||
"https://www.youtube.com/watch?v=abc",
|
||||
"https://youtu.be/abc",
|
||||
"https://music.youtube.com/watch?v=abc",
|
||||
"https://www.youtube-nocookie.com/embed/abc",
|
||||
):
|
||||
assert credentials.host_key_for_url(url) == "youtube", url
|
||||
|
||||
def test_look_alike_host_never_matches(self):
|
||||
assert credentials.host_key_for_url("https://myfacebook.com/x") is None
|
||||
assert credentials.host_key_for_url("https://facebook.com.evil.test/x") is None
|
||||
assert credentials.host_key_for_url("https://notyoutube.com/watch?v=abc") is None
|
||||
assert credentials.host_key_for_url("https://youtube.com.evil.test/x") is None
|
||||
|
||||
def test_unrelated_or_empty_url_has_no_host_key(self):
|
||||
assert credentials.host_key_for_url("https://www.youtube.com/watch?v=abc") is None
|
||||
assert credentials.host_key_for_url("https://vimeo.com/123456") is None
|
||||
assert credentials.host_key_for_url("") is None
|
||||
assert credentials.host_key_for_url(None) is None
|
||||
|
||||
@@ -72,6 +82,21 @@ def test_authed_sig_differs_per_user_and_from_the_shared_sig():
|
||||
assert a != shared and b != shared and a != b
|
||||
|
||||
|
||||
def test_an_authed_asset_queued_before_a_signature_version_bump_still_gets_its_cookie(monkeypatch):
|
||||
# A job keeps the asset it was queued with. A deploy that bumps the signature version between the
|
||||
# queueing and the download used to make the user's private asset stop counting as theirs, so the
|
||||
# download went without its cookie and failed on the login wall.
|
||||
spec = formats.normalize({"mode": "a"})
|
||||
queued = MediaAsset(format_sig=credentials.authed_format_sig(spec, 1, "facebook"))
|
||||
rebuild = credentials.rebuild_salt(7, credentials.cookie_salt(1, "facebook"))
|
||||
rebuilt = MediaAsset(id=7, format_sig=formats.format_sig(spec, salt=rebuild))
|
||||
monkeypatch.setattr(formats, "_AUDIO_SIG_VERSION", formats._AUDIO_SIG_VERSION + 1) # the deploy
|
||||
|
||||
for asset in (queued, rebuilt):
|
||||
assert credentials.is_authed_asset(asset, spec, 1, FB_URL)
|
||||
assert not credentials.is_authed_asset(asset, spec, 2, FB_URL) # still private to its user
|
||||
|
||||
|
||||
def test_write_cookiefile_is_private_and_survives_the_retry_wipe(tmp_path):
|
||||
staging = tmp_path / "staging"
|
||||
staging.mkdir()
|
||||
@@ -160,6 +185,22 @@ class TestCachePrivacy:
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
assert asset.format_sig == formats.format_sig(formats.normalize({}))
|
||||
|
||||
def test_a_youtube_job_reaches_the_gate_through_its_rebuilt_watch_url(self, db):
|
||||
# The one join nothing else makes. A YouTube job stores a bare 11-char ref, and the URL the
|
||||
# cookie gate keys on is REBUILT by the adapter (service.source_url -> YouTubeAdapter). Both
|
||||
# ends are pinned apart — the watch-URL shape in test_adapters, the host_key in
|
||||
# TestHostKeyForUrl — so this welds them: an enqueue that ever handed the gate the raw ref
|
||||
# would silently stop salting, every YouTube cookie would quietly stop being attached, and
|
||||
# signed-out downloads would keep working, so nothing else here would go red.
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "youtube", COOKIES)
|
||||
db.commit()
|
||||
job = service.enqueue(db, u.id, "youtube", "VIDID12345_", {})
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
spec = formats.normalize({})
|
||||
assert asset.format_sig == credentials.authed_format_sig(spec, u.id, "youtube")
|
||||
assert asset.format_sig != formats.format_sig(spec)
|
||||
|
||||
def test_two_users_never_share_the_authed_asset(self, db):
|
||||
a = _user(db, "a@example.com")
|
||||
b = _user(db, "b@example.com")
|
||||
@@ -189,7 +230,9 @@ class TestResolveCookiefile:
|
||||
db.commit()
|
||||
spec = formats.normalize({})
|
||||
salted = credentials.authed_format_sig(spec, u.id, "facebook")
|
||||
path = credentials.resolve_cookiefile(db, u.id, FB_URL, salted, spec, tmp_path)
|
||||
path = credentials.resolve_cookiefile(
|
||||
db, u.id, FB_URL, self._asset_sig(salted), spec, tmp_path
|
||||
)
|
||||
assert path is not None and Path(path).read_text() == COOKIES
|
||||
|
||||
def test_never_attaches_to_a_shared_asset(self, db, tmp_path):
|
||||
@@ -200,23 +243,176 @@ class TestResolveCookiefile:
|
||||
db.commit()
|
||||
spec = formats.normalize({})
|
||||
shared = formats.format_sig(spec)
|
||||
assert credentials.resolve_cookiefile(db, u.id, FB_URL, shared, spec, tmp_path) is None
|
||||
assert (
|
||||
credentials.resolve_cookiefile(
|
||||
db, u.id, FB_URL, self._asset_sig(shared), spec, tmp_path
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_non_matching_host_gets_no_cookie(self, db, tmp_path):
|
||||
u = _user(db)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
spec = formats.normalize({})
|
||||
# A YouTube URL has no supported host_key -> no cookie regardless of what's stored.
|
||||
yt = "https://www.youtube.com/watch?v=abc"
|
||||
# A URL of ANOTHER host (supported or not) never gets this credential's cookie. YouTube is
|
||||
# a supported host in its own right now, so the case needs a host that is not.
|
||||
other = "https://vimeo.com/123456"
|
||||
salted = credentials.authed_format_sig(spec, u.id, "facebook")
|
||||
assert credentials.resolve_cookiefile(db, u.id, yt, salted, spec, tmp_path) is None
|
||||
assert (
|
||||
credentials.resolve_cookiefile(db, u.id, other, self._asset_sig(salted), spec, tmp_path)
|
||||
is None
|
||||
)
|
||||
# ...and a YouTube link gets nothing either: the stored credential is Facebook's.
|
||||
yt = "https://www.youtube.com/watch?v=abc"
|
||||
assert (
|
||||
credentials.resolve_cookiefile(db, u.id, yt, self._asset_sig(salted), spec, tmp_path)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_no_credential_gets_no_cookie(self, db, tmp_path):
|
||||
u = _user(db)
|
||||
spec = formats.normalize({})
|
||||
salted = credentials.authed_format_sig(spec, u.id, "facebook")
|
||||
assert credentials.resolve_cookiefile(db, u.id, FB_URL, salted, spec, tmp_path) is None
|
||||
assert (
|
||||
credentials.resolve_cookiefile(
|
||||
db, u.id, FB_URL, self._asset_sig(salted), spec, tmp_path
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
# --- Retry / Re-download take a cookie added later (DB) -----------------------------------------
|
||||
# Measured on dev (E11.5): a Facebook reel failed on the login wall, the user added their cookie as
|
||||
# the failure copy tells them to, pressed Retry — and it failed again, because the job still held the
|
||||
# shared asset the worker never attaches a cookie to. Re-download forked a salt the gate could not
|
||||
# recognise, so even a job downloaded WITH a cookie rebuilt without it.
|
||||
|
||||
|
||||
def _held_job(db, u, *, sig, asset_status="error", job_status="error"):
|
||||
a = MediaAsset(
|
||||
source_kind="url", source_ref=FB_URL, format_sig=sig, status=asset_status, ref_count=1
|
||||
)
|
||||
db.add(a)
|
||||
db.flush()
|
||||
j = DownloadJob(
|
||||
user_id=u.id,
|
||||
asset_id=a.id,
|
||||
source_kind="url",
|
||||
source_ref=FB_URL,
|
||||
status=job_status,
|
||||
queue_pos=0,
|
||||
profile_snapshot={},
|
||||
)
|
||||
db.add(j)
|
||||
db.commit()
|
||||
return a, j
|
||||
|
||||
|
||||
class TestRequeueTakesTheCookie:
|
||||
spec = formats.normalize({})
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("asset_status", "job_status"),
|
||||
# A failed download; and a job paused before it ever started (its asset never downloaded).
|
||||
[("error", "error"), ("pending", "paused")],
|
||||
)
|
||||
def test_retry_after_adding_a_cookie_moves_the_job_to_the_private_asset(
|
||||
self, db, tmp_path, asset_status, job_status
|
||||
):
|
||||
u = _user(db)
|
||||
shared, j = _held_job(
|
||||
db,
|
||||
u,
|
||||
sig=formats.format_sig(self.spec),
|
||||
asset_status=asset_status,
|
||||
job_status=job_status,
|
||||
)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
|
||||
service.resume(db, j)
|
||||
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, j.id)
|
||||
target = db.get(MediaAsset, job.asset_id)
|
||||
assert job.status == "queued" and target.id != shared.id
|
||||
assert target.format_sig == credentials.authed_format_sig(self.spec, u.id, "facebook")
|
||||
assert (target.status, target.ref_count) == ("pending", 1)
|
||||
assert db.get(MediaAsset, shared.id).ref_count == 0 # the old hold went with the move
|
||||
assert credentials.resolve_cookiefile(db, u.id, FB_URL, target, self.spec, tmp_path)
|
||||
|
||||
def test_retry_without_a_cookie_keeps_the_shared_asset(self, db):
|
||||
u = _user(db)
|
||||
shared, j = _held_job(db, u, sig=formats.format_sig(self.spec))
|
||||
|
||||
service.resume(db, j)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, j.id).asset_id == shared.id
|
||||
assert db.get(MediaAsset, shared.id).ref_count == 1
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("asset_status", "job_status"), [("ready", "error"), ("downloading", "paused")]
|
||||
)
|
||||
def test_retry_never_moves_off_a_file_or_a_download_in_flight(
|
||||
self, db, asset_status, job_status
|
||||
):
|
||||
u = _user(db)
|
||||
shared, j = _held_job(
|
||||
db,
|
||||
u,
|
||||
sig=formats.format_sig(self.spec),
|
||||
asset_status=asset_status,
|
||||
job_status=job_status,
|
||||
)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
|
||||
service.resume(db, j)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, j.id).asset_id == shared.id
|
||||
|
||||
def test_redownload_with_a_cookie_rebuilds_into_an_asset_the_cookie_goes_with(
|
||||
self, db, tmp_path
|
||||
):
|
||||
u = _user(db)
|
||||
other = _user(db, "other@example.com")
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
credentials.upsert_credential(db, other.id, "facebook", COOKIES)
|
||||
authed = credentials.authed_format_sig(self.spec, u.id, "facebook")
|
||||
first, j = _held_job(db, u, sig=authed, asset_status="ready", job_status="done")
|
||||
|
||||
seen = {first.id}
|
||||
for _ in range(2): # a rebuild of a rebuild is still recognised
|
||||
service.redownload(db, j)
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, j.id)
|
||||
target = db.get(MediaAsset, job.asset_id)
|
||||
assert target.id not in seen and target.format_sig != authed
|
||||
seen.add(target.id)
|
||||
assert credentials.resolve_cookiefile(db, u.id, FB_URL, target, self.spec, tmp_path)
|
||||
# Private: another user's cookie is never attached to it.
|
||||
assert not credentials.is_authed_asset(target, self.spec, other.id, FB_URL)
|
||||
|
||||
# A Retry of a failed rebuild keeps the rebuild — it is already the cookie's identity.
|
||||
target.status, job.status = "error", "error"
|
||||
db.commit()
|
||||
service.resume(db, job)
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, j.id).asset_id == target.id
|
||||
|
||||
def test_a_rebuild_made_without_a_cookie_is_no_cookie_identity(self, db, tmp_path):
|
||||
u = _user(db)
|
||||
_, j = _held_job(db, u, sig=formats.format_sig(self.spec))
|
||||
service.redownload(db, j)
|
||||
credentials.upsert_credential(db, u.id, "facebook", COOKIES)
|
||||
db.commit()
|
||||
|
||||
db.expire_all()
|
||||
target = db.get(MediaAsset, db.get(DownloadJob, j.id).asset_id)
|
||||
assert not credentials.is_authed_asset(target, self.spec, u.id, FB_URL)
|
||||
|
||||
|
||||
# --- API: write-only, never echoes the blob (DB) ----------------------------------------------
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Category classification of raw yt-dlp / ffmpeg download-failure strings (S2)."""
|
||||
|
||||
from app.downloads import adapters
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads.errors import classify
|
||||
|
||||
@@ -64,6 +65,164 @@ def test_our_own_worker_refusals_classify_as_themselves():
|
||||
assert classify(dlerrors.QUOTA_FULL_MESSAGE) == "quota_exceeded"
|
||||
|
||||
|
||||
def test_the_redacted_network_text_classifies_back_to_network():
|
||||
# The redacted text is also what a failed asset keeps, and a second job that meets that asset is
|
||||
# diagnosed from it — without its own rule it would read "Download failed".
|
||||
assert classify(dlerrors.NETWORK_REDACTED_MESSAGE) == "network"
|
||||
|
||||
|
||||
# Verbatim yt-dlp 2026.07.04 output from prod jobs, measured on dev 2026-09-13 (E11.5).
|
||||
_FB_PARSE = (
|
||||
"ERROR: [facebook] 1736360027626346: Cannot parse data; please report this issue on "
|
||||
"https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template."
|
||||
)
|
||||
_FB_NO_FORMATS = (
|
||||
"ERROR: [facebook] 1755988225525547: No video formats found!; please report this issue on "
|
||||
"https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template."
|
||||
)
|
||||
_REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
|
||||
_GROUP_POST = "https://www.facebook.com/groups/983554479435596/permalink/1755988225525547/?"
|
||||
|
||||
|
||||
# Verbatim from prod job 273 (2026-09-15). YouTube writes the apostrophe as U+2019, which is why
|
||||
# neither the rule nor the catalog entry may spell out "you’re" with a straight quote.
|
||||
_YT_BOT_WALL = (
|
||||
"ERROR: [youtube] 7JPmS_bWjhI: Sign in to confirm you’re not a bot. Use "
|
||||
"--cookies-from-browser or --cookies for the authentication. See "
|
||||
"https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for how to manually "
|
||||
"pass cookies."
|
||||
)
|
||||
_YT = "https://www.youtube.com/watch?v=7JPmS_bWjhI"
|
||||
|
||||
|
||||
def test_youtube_bot_wall_is_that_video_own_signed_out_gate():
|
||||
# Measured on prod: all 12 player clients fail on this video while its neighbours download, the
|
||||
# POT sidecar answers, and the newest yt-dlp fails identically — a cookie is the only way in.
|
||||
assert classify(_YT_BOT_WALL) == "login_required"
|
||||
for url in (
|
||||
_YT,
|
||||
"https://youtu.be/7JPmS_bWjhI",
|
||||
"https://www.youtube-nocookie.com/embed/7JPmS_bWjhI",
|
||||
"https://music.youtube.com/watch?v=7JPmS_bWjhI",
|
||||
):
|
||||
assert dlerrors.diagnose(_YT_BOT_WALL, url=url) == (
|
||||
"login_required",
|
||||
"youtube.bot_wall",
|
||||
), url
|
||||
|
||||
|
||||
def test_youtube_bot_wall_with_a_cookie_is_the_cookie_refused():
|
||||
# Telling a user who already gave a cookie to add one is what this entry exists to avoid.
|
||||
assert dlerrors.diagnose(_YT_BOT_WALL, url=_YT, cookie_attached=True) == (
|
||||
"login_required",
|
||||
"youtube.cookie_rejected",
|
||||
)
|
||||
|
||||
|
||||
# Verbatim from the isolated test stack (2026-09-16), the SAME video ~90 minutes later: yt-dlp got
|
||||
# the player response and picked formats, and the CDN then refused the stream.
|
||||
_YT_MEDIA_403 = "ERROR: unable to download video data: HTTP Error 403: Forbidden"
|
||||
|
||||
|
||||
def test_youtube_media_403_is_the_wall_one_step_later_not_a_network_fault():
|
||||
# Generically this text is a network failure — and for every other host it stays one.
|
||||
assert classify(_YT_MEDIA_403) == "network"
|
||||
assert dlerrors.diagnose(_YT_MEDIA_403, url="https://vimeo.com/1") == ("network", None)
|
||||
# On a signed-out YouTube download it is the gate: measured, the same video downloaded in full
|
||||
# once the owner's cookie was attached. Calling that "network error" is the honest-states bug.
|
||||
assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT) == (
|
||||
"login_required",
|
||||
"youtube.media_forbidden",
|
||||
)
|
||||
|
||||
|
||||
def test_youtube_media_403_with_a_cookie_is_not_claimed_as_the_gate():
|
||||
# Only the cookie-less case was measured. With a cookie the same text keeps its bare category
|
||||
# rather than telling a signed-in user to sign in.
|
||||
assert dlerrors.diagnose(_YT_MEDIA_403, url=_YT, cookie_attached=True) == ("network", None)
|
||||
|
||||
|
||||
def test_the_youtube_entries_never_match_another_site():
|
||||
# Host-scoped: the same words about a Facebook link stay the bare category.
|
||||
assert dlerrors.diagnose(
|
||||
"ERROR: [facebook] 1: Sign in to confirm you’re not a bot",
|
||||
url="https://www.facebook.com/reel/1/",
|
||||
) == ("login_required", None)
|
||||
|
||||
|
||||
def test_facebook_cannot_parse_data_is_the_login_wall_without_a_cookie():
|
||||
for url in (
|
||||
_REEL,
|
||||
_GROUP_POST,
|
||||
"https://fb.watch/Ia_iJYI2Vw/?fs=e",
|
||||
"https://m.facebook.com/x",
|
||||
):
|
||||
assert dlerrors.diagnose(_FB_PARSE, url=url) == ("login_required", "facebook.login_wall")
|
||||
|
||||
|
||||
def test_facebook_cannot_parse_data_with_a_cookie_is_the_cookie_refused():
|
||||
# Telling a user who already gave a cookie to add one is the failure this entry exists to avoid.
|
||||
assert dlerrors.diagnose(_FB_PARSE, url=_REEL, cookie_attached=True) == (
|
||||
"login_required",
|
||||
"facebook.cookie_rejected",
|
||||
)
|
||||
|
||||
|
||||
def test_a_facebook_group_post_without_formats_names_the_link():
|
||||
# Without a cookie yt-dlp is given the post link itself (measured, prod jobs 66 and 145).
|
||||
assert dlerrors.diagnose(_FB_NO_FORMATS, url=_GROUP_POST) == (
|
||||
"format_unavailable",
|
||||
"facebook.group_post_video",
|
||||
)
|
||||
# With one it is only ever given the post's VIDEO link, so the text is about that video: the adapter
|
||||
# found it, and "this post has no video" would be false.
|
||||
assert dlerrors.diagnose(_FB_NO_FORMATS, url=_GROUP_POST, cookie_attached=True) == (
|
||||
"format_unavailable",
|
||||
None,
|
||||
)
|
||||
# A reel, or a group link that is not a post, is another failure: only the category.
|
||||
for url in (_REEL, "https://www.facebook.com/groups/983554479435596/"):
|
||||
assert dlerrors.diagnose(_FB_NO_FORMATS, url=url) == ("format_unavailable", None), url
|
||||
|
||||
|
||||
def test_our_post_not_shown_verdict_is_the_refused_cookie():
|
||||
# The adapter's text when a signed-in read of the post got a page without the post (the login page).
|
||||
msg = adapters.POST_NOT_SHOWN_MESSAGE
|
||||
assert classify(msg) == "login_required"
|
||||
assert dlerrors.diagnose(msg, url=_GROUP_POST, cookie_attached=True) == (
|
||||
"login_required",
|
||||
"facebook.cookie_rejected",
|
||||
)
|
||||
|
||||
|
||||
def test_our_post_without_a_video_verdict_names_the_group_link():
|
||||
# The adapter's own text when the post page holds no single video (S4): the message and its rule and
|
||||
# catalog entry are pinned together, or an edit to the wording orphans the copy.
|
||||
msg = adapters.POST_WITHOUT_VIDEO_MESSAGE
|
||||
assert classify(msg) == "format_unavailable"
|
||||
assert dlerrors.diagnose(msg, url=_GROUP_POST, cookie_attached=True) == (
|
||||
"format_unavailable",
|
||||
"facebook.group_post_video",
|
||||
)
|
||||
assert dlerrors.diagnose(msg, url=_REEL, cookie_attached=True) == ("format_unavailable", None)
|
||||
|
||||
|
||||
def test_a_catalog_entry_never_leaves_its_host():
|
||||
# "Cannot parse data" is yt-dlp's wording for a broken extractor on every site.
|
||||
other = _FB_PARSE.replace("[facebook]", "[vimeo]")
|
||||
assert dlerrors.diagnose(other, url="https://vimeo.com/1") == ("unknown", None)
|
||||
# The Facebook text for a link that is not Facebook's (a look-alike host) is not the Facebook wall.
|
||||
assert dlerrors.diagnose(_FB_PARSE, url="https://facebook.com.evil/reel/1") == ("unknown", None)
|
||||
# No link known: the category only.
|
||||
assert dlerrors.diagnose(_FB_PARSE) == ("unknown", None)
|
||||
|
||||
|
||||
def test_diagnose_falls_back_to_the_category():
|
||||
msg = "ERROR: [facebook] 123: You must be logged in to see this post"
|
||||
assert dlerrors.diagnose(msg, url=_REEL) == ("login_required", None)
|
||||
assert dlerrors.diagnose(None, url=_REEL) == ("unknown", None)
|
||||
|
||||
|
||||
def test_a_url_jobs_network_failure_is_redacted_but_youtube_keeps_its_text():
|
||||
# SEC-12: the raw transport text for a user-chosen host:port is a port-scanner readout, so it is
|
||||
# replaced for `url` jobs. Everything else must survive verbatim, or a real diagnosis is lost.
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
"""DB-backed tests for the download failure catalog (E11.5 S1) at the places a failure is WRITTEN.
|
||||
|
||||
`test_download_errors` pins which entry a message matches; these pin that every writer hands the
|
||||
matcher what it needs — the link and whether the user's cookie went with it — and that a stored entry
|
||||
never outlives the failure it describes:
|
||||
- the download child's failure branch (`pipeline._download`);
|
||||
- a second job meeting an already-failed asset (`worker._process_job`), which only has the text;
|
||||
- Retry / Re-download, which requeue the job;
|
||||
- the `0084` backfill of the rows that failed before the catalog existed;
|
||||
- the downloads list, which is how the entry reaches the UI.
|
||||
It also holds the one success-path check of the same child: the duration it stores (E11.5 S3).
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from cryptography.fernet import Fernet
|
||||
from sqlalchemy import text
|
||||
|
||||
from app import security
|
||||
from app.config import settings
|
||||
from app.db import engine
|
||||
from app.downloads import credentials, formats, pipeline, service
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset, User
|
||||
from app.worker import _process_job
|
||||
|
||||
_REEL = "https://www.facebook.com/reel/1736360027626346/?fs=e&fs=e"
|
||||
# Verbatim yt-dlp 2026.07.04, prod job 256.
|
||||
_FB_PARSE = (
|
||||
"ERROR: [facebook] 1736360027626346: Cannot parse data; please report this issue on "
|
||||
"https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template."
|
||||
)
|
||||
_COOKIES = "# Netscape HTTP Cookie File\n.facebook.com\tTRUE\t/\tTRUE\t0\tc_user\t123\n"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _fernet(monkeypatch):
|
||||
# A throwaway key, so a stored credential encrypts and decrypts (mirrors test_download_credentials).
|
||||
monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key()))
|
||||
|
||||
|
||||
def _user(db, email):
|
||||
u = User(email=email)
|
||||
db.add(u)
|
||||
db.flush()
|
||||
db.add(
|
||||
DownloadQuota(
|
||||
user_id=u.id, max_bytes=10**12, max_concurrent=5, max_jobs=10**6, unlimited=True
|
||||
)
|
||||
)
|
||||
return u
|
||||
|
||||
|
||||
def _asset_and_job(db, user, *, sig, asset_status, job_status, error=None):
|
||||
a = MediaAsset(
|
||||
source_kind="url",
|
||||
source_ref=_REEL,
|
||||
format_sig=sig,
|
||||
status=asset_status,
|
||||
error=error,
|
||||
ref_count=1,
|
||||
)
|
||||
db.add(a)
|
||||
db.flush()
|
||||
j = DownloadJob(
|
||||
user_id=user.id,
|
||||
asset_id=a.id,
|
||||
source_kind="url",
|
||||
source_ref=_REEL,
|
||||
status=job_status,
|
||||
queue_pos=0,
|
||||
profile_snapshot={},
|
||||
)
|
||||
db.add(j)
|
||||
db.commit()
|
||||
return a, j
|
||||
|
||||
|
||||
def _fail_the_child(monkeypatch, tmp_path, *, cookie: bool):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
cookiefile = tmp_path / "cookies.txt"
|
||||
cookiefile.write_text("# Netscape HTTP Cookie File\n")
|
||||
monkeypatch.setattr(
|
||||
pipeline.credentials,
|
||||
"resolve_cookiefile",
|
||||
lambda *a, **k: cookiefile if cookie else None,
|
||||
)
|
||||
|
||||
def extract(*a, **k):
|
||||
raise RuntimeError(_FB_PARSE)
|
||||
|
||||
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||
|
||||
|
||||
def test_the_download_child_diagnoses_with_the_link_and_no_cookie(db, monkeypatch, tmp_path):
|
||||
u = _user(db, "fb-nocookie@example.com")
|
||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||
_fail_the_child(monkeypatch, tmp_path, cookie=False)
|
||||
|
||||
pipeline._download(j.id, a.id, "url", _REEL, {}, u.id)
|
||||
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, j.id)
|
||||
assert (job.status, job.error_code, job.error_detail) == (
|
||||
"error",
|
||||
"login_required",
|
||||
"facebook.login_wall",
|
||||
)
|
||||
|
||||
|
||||
def test_the_download_child_knows_the_cookie_went_with_it(db, monkeypatch, tmp_path):
|
||||
# The same text with the user's cookie attached: telling them to add cookies would be wrong.
|
||||
u = _user(db, "fb-cookie@example.com")
|
||||
a, j = _asset_and_job(db, u, sig="private", asset_status="downloading", job_status="running")
|
||||
_fail_the_child(monkeypatch, tmp_path, cookie=True)
|
||||
|
||||
pipeline._download(j.id, a.id, "url", _REEL, {}, u.id)
|
||||
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, j.id)
|
||||
assert (job.error_code, job.error_detail) == ("login_required", "facebook.cookie_rejected")
|
||||
|
||||
|
||||
def test_a_job_meeting_a_failed_shared_asset_gets_the_same_explanation(db):
|
||||
# Before the catalog this path wrote the text and NO code: "Download failed" for the second
|
||||
# job, "Facebook sign-in required" for the first.
|
||||
u = _user(db, "fb-second@example.com")
|
||||
_, j = _asset_and_job(
|
||||
db, u, sig="shared", asset_status="error", job_status="running", error=_FB_PARSE
|
||||
)
|
||||
|
||||
_process_job(j.id)
|
||||
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, j.id)
|
||||
assert (job.status, job.error_code, job.error_detail) == (
|
||||
"error",
|
||||
"login_required",
|
||||
"facebook.login_wall",
|
||||
)
|
||||
|
||||
|
||||
def test_a_job_meeting_its_users_failed_private_asset_knows_the_cookie_was_used(db, _fernet):
|
||||
u = _user(db, "fb-private@example.com")
|
||||
credentials.upsert_credential(db, u.id, "facebook", _COOKIES)
|
||||
sig = credentials.authed_format_sig(formats.normalize({}), u.id, "facebook")
|
||||
_, j = _asset_and_job(
|
||||
db, u, sig=sig, asset_status="error", job_status="running", error=_FB_PARSE
|
||||
)
|
||||
|
||||
_process_job(j.id)
|
||||
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, j.id)
|
||||
assert (job.error_code, job.error_detail) == ("login_required", "facebook.cookie_rejected")
|
||||
|
||||
|
||||
def test_a_private_asset_the_cookie_could_not_go_with_is_explained_as_the_download_had_it(db):
|
||||
# The cookie was deleted (or can no longer be decrypted) after the jobs were queued: the download
|
||||
# child had nothing to attach and said "add your cookies". The job meeting that failed asset must say
|
||||
# the same — the asset's salt alone would have told it "your cookies were refused".
|
||||
u = _user(db, "fb-gone@example.com")
|
||||
sig = credentials.authed_format_sig(formats.normalize({}), u.id, "facebook")
|
||||
_, j = _asset_and_job(
|
||||
db, u, sig=sig, asset_status="error", job_status="running", error=_FB_PARSE
|
||||
)
|
||||
|
||||
_process_job(j.id)
|
||||
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, j.id)
|
||||
assert (job.error_code, job.error_detail) == ("login_required", "facebook.login_wall")
|
||||
|
||||
|
||||
def test_a_job_canceled_before_it_meets_a_failed_asset_stays_canceled(db):
|
||||
# The cancel landed between the claim and this write. `error` would hold the asset again after the
|
||||
# route already released it — a later delete releases twice.
|
||||
u = _user(db, "fb-canceled@example.com")
|
||||
_, j = _asset_and_job(
|
||||
db, u, sig="shared", asset_status="error", job_status="canceled", error=_FB_PARSE
|
||||
)
|
||||
|
||||
_process_job(j.id)
|
||||
|
||||
db.expire_all()
|
||||
job = db.get(DownloadJob, j.id)
|
||||
assert (job.status, job.error_code, job.error_detail) == ("canceled", None, None)
|
||||
|
||||
|
||||
def test_the_downloads_list_carries_the_catalog_entry(db, as_user):
|
||||
# The whole explanation rides on this one field: without it every row falls back to the category.
|
||||
u = _user(db, "fb-list@example.com")
|
||||
_, j = _asset_and_job(
|
||||
db, u, sig="shared", asset_status="error", job_status="error", error=_FB_PARSE
|
||||
)
|
||||
j.error_code, j.error_detail = "login_required", "facebook.login_wall"
|
||||
db.commit()
|
||||
|
||||
[row] = as_user(u).get("/api/downloads").json()
|
||||
|
||||
assert (row["error_code"], row["error_detail"]) == ("login_required", "facebook.login_wall")
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
def test_a_download_yt_dlp_gave_no_duration_stores_the_files_duration(db, monkeypatch, tmp_path):
|
||||
# Facebook reel 257 (E11.5): yt-dlp's info had no duration, the asset stored NULL, and the Library
|
||||
# row lost its trim action. Through the real finalize, with a real 3-second file.
|
||||
u = _user(db, "fb-duration@example.com")
|
||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
|
||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||
media = staging / "1736360027626346.mp4"
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc=size=64x64:rate=10",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440",
|
||||
"-t",
|
||||
"3",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"aac",
|
||||
str(media),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
return {"id": "1736360027626346", "ext": "mp4", "title": "Reel", "filepath": str(media)}
|
||||
|
||||
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||
|
||||
pipeline._download(j.id, a.id, "url", _REEL, {}, u.id)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, j.id).status == "done"
|
||||
assert db.get(MediaAsset, a.id).duration_s == 3
|
||||
|
||||
|
||||
def test_retry_and_redownload_drop_the_old_explanation(db):
|
||||
u = _user(db, "fb-requeue@example.com")
|
||||
for requeue in (service.resume, service.redownload):
|
||||
_, j = _asset_and_job(
|
||||
db, u, sig=f"s-{requeue.__name__}", asset_status="error", job_status="error"
|
||||
)
|
||||
j.error, j.error_code, j.error_detail = _FB_PARSE, "login_required", "facebook.login_wall"
|
||||
db.commit()
|
||||
|
||||
requeue(db, j)
|
||||
|
||||
db.refresh(j)
|
||||
assert (j.status, j.error_code, j.error_detail) == ("queued", None, None), requeue.__name__
|
||||
|
||||
|
||||
def test_0084_backfills_only_the_facebook_parse_failures_it_measured(db):
|
||||
cfg = Config("alembic.ini")
|
||||
command.downgrade(cfg, "0083_comfort_pins")
|
||||
upgraded = False
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
uid = conn.execute(
|
||||
text("INSERT INTO users (email) VALUES ('backfill@example.com') RETURNING id")
|
||||
).scalar_one()
|
||||
rows = {
|
||||
"fb_unknown": ("error", "unknown", _FB_PARSE),
|
||||
"fb_null": ("canceled", None, _FB_PARSE),
|
||||
"other_site": ("error", "unknown", _FB_PARSE.replace("[facebook]", "[vimeo]")),
|
||||
"fb_other_code": ("error", "network", _FB_PARSE),
|
||||
}
|
||||
ids = {}
|
||||
for name, (status, code, error) in rows.items():
|
||||
ids[name] = conn.execute(
|
||||
text(
|
||||
"INSERT INTO download_jobs (user_id, source_kind, source_ref, "
|
||||
"profile_snapshot, status, error_code, error) "
|
||||
"VALUES (:u, 'url', :ref, '{}', :s, :c, :e) RETURNING id"
|
||||
),
|
||||
{"u": uid, "ref": _REEL, "s": status, "c": code, "e": error},
|
||||
).scalar_one()
|
||||
command.upgrade(cfg, "head")
|
||||
upgraded = True
|
||||
with engine.connect() as conn:
|
||||
got = {
|
||||
name: tuple(
|
||||
conn.execute(
|
||||
text("SELECT error_code, error_detail FROM download_jobs WHERE id = :i"),
|
||||
{"i": i},
|
||||
).one()
|
||||
)
|
||||
for name, i in ids.items()
|
||||
}
|
||||
assert got == {
|
||||
"fb_unknown": ("login_required", "facebook.login_wall"),
|
||||
"fb_null": ("login_required", "facebook.login_wall"),
|
||||
"other_site": ("unknown", None),
|
||||
"fb_other_code": ("network", None),
|
||||
}
|
||||
finally:
|
||||
if not upgraded:
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
|
||||
def _ffmpeg_media(path, *codec_args):
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-loglevel", "error", "-f", "lavfi", "-i", "sine=frequency=440"]
|
||||
+ list(codec_args)
|
||||
+ ["-t", "2", str(path)],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
def test_a_signed_in_facebook_download_is_titled_with_its_posts_text(db, monkeypatch, tmp_path):
|
||||
# Prod jobs 265–268 (2026-09-14): with a cookie yt-dlp titles every reel "Video", so the row, the
|
||||
# file name and the share preview all said "Video". The post's text is in the description.
|
||||
u = _user(db, "fb-title@example.com")
|
||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
|
||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||
media = staging / "2131884261040652.m4a"
|
||||
_ffmpeg_media(media, "-c:a", "aac")
|
||||
return {
|
||||
"id": "2131884261040652",
|
||||
"ext": "m4a",
|
||||
"title": "Video",
|
||||
"description": "Tune of The Day, Argy playing his version",
|
||||
"extractor_key": "Facebook",
|
||||
"filepath": str(media),
|
||||
}
|
||||
|
||||
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||
|
||||
pipeline._download(j.id, a.id, "url", _REEL, {"mode": "a"}, u.id)
|
||||
|
||||
db.expire_all()
|
||||
asset, job = db.get(MediaAsset, a.id), db.get(DownloadJob, j.id)
|
||||
assert asset.title == job.display_name == "Tune of The Day, Argy playing his version"
|
||||
assert "Tune_of_The_Day" in asset.rel_path
|
||||
|
||||
|
||||
def test_the_early_title_takes_the_posts_text_too(db):
|
||||
u = _user(db, "fb-early-title@example.com")
|
||||
a, _ = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||
a.title = None
|
||||
db.commit()
|
||||
|
||||
pipeline._fill_asset_meta_early(
|
||||
a.id,
|
||||
{
|
||||
"id": "1",
|
||||
"title": "Video",
|
||||
"description": "Almás palacsinta",
|
||||
"extractor_key": "Facebook",
|
||||
},
|
||||
)
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(MediaAsset, a.id).title == "Almás palacsinta"
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
def test_an_mp3_download_is_stored_as_an_mp3(db, monkeypatch, tmp_path):
|
||||
# Prod job 268: the "Audio (MP3)" profile's extracted .mp3 was remuxed into an .mp4 and served as
|
||||
# video/mp4, which iPad's WebKit will not play. It stays an .mp3 (served as audio/mpeg).
|
||||
u = _user(db, "fb-mp3@example.com")
|
||||
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
|
||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||
media = staging / "2131884261040652.mp3"
|
||||
_ffmpeg_media(media, "-c:a", "libmp3lame")
|
||||
return {"id": "2131884261040652", "ext": "mp3", "title": "Reel", "filepath": str(media)}
|
||||
|
||||
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||
|
||||
pipeline._download(
|
||||
j.id, a.id, "url", _REEL, formats.normalize({"mode": "a", "audio_format": "mp3"}), u.id
|
||||
)
|
||||
|
||||
db.expire_all()
|
||||
asset = db.get(MediaAsset, a.id)
|
||||
assert (asset.container, asset.acodec, asset.media_kind) == ("mp3", "mp3", "audio")
|
||||
assert asset.rel_path.endswith(".mp3")
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available")
|
||||
@pytest.mark.parametrize("mode", ["av", "v"])
|
||||
def test_a_video_download_is_still_made_browser_playable(db, monkeypatch, tmp_path, mode):
|
||||
# The mp3 fix skips the browser-playable pass for AUDIO only: a video that arrives in a container
|
||||
# browsers won't play (an .mkv) is still remuxed into an .mp4.
|
||||
u = _user(db, f"fb-mkv-{mode}@example.com")
|
||||
a, j = _asset_and_job(
|
||||
db, u, sig=f"mkv-{mode}", asset_status="downloading", job_status="running"
|
||||
)
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
|
||||
|
||||
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
|
||||
media = staging / "1736360027626346.mkv"
|
||||
audio = ["-f", "lavfi", "-i", "sine=frequency=440", "-c:a", "aac"] if mode == "av" else []
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc=size=64x64:rate=10",
|
||||
]
|
||||
+ audio
|
||||
+ ["-t", "1", "-c:v", "libx264", "-pix_fmt", "yuv420p", str(media)],
|
||||
check=True,
|
||||
)
|
||||
return {"id": "1736360027626346", "ext": "mkv", "title": "Reel", "filepath": str(media)}
|
||||
|
||||
monkeypatch.setattr(pipeline, "_extract_with_fallback", extract)
|
||||
|
||||
pipeline._download(j.id, a.id, "url", _REEL, formats.normalize({"mode": mode}), u.id)
|
||||
|
||||
db.expire_all()
|
||||
asset = db.get(MediaAsset, a.id)
|
||||
assert (asset.container, asset.vcodec) == ("mp4", "h264")
|
||||
assert asset.rel_path.endswith(".mp4")
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Overlapping requests on ONE job (E11.5 review) — DB-backed, two real sessions.
|
||||
|
||||
The routes that move a job's asset ref count (resume, cancel, delete, redownload) used to decide on an
|
||||
unlocked read: two overlapping requests for one job (a double-click, a second tab, a replay after a
|
||||
gateway timeout) both saw it holding its asset and both released the hold. The count then reaches 0
|
||||
while another job still holds the asset, and GC or the ready-file delete removes it under that job.
|
||||
A pause racing a cancel did the same one step later: its unlocked write turned the released job back
|
||||
into a holder, which a later delete releases again. `_own_job_for_update` makes the second request
|
||||
wait and decide on the first one's committed result.
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import service
|
||||
from app.models import DownloadJob, MediaAsset, User
|
||||
from app.routes.downloads import (
|
||||
_own_job_for_update,
|
||||
cancel_download,
|
||||
delete_download,
|
||||
pause_download,
|
||||
)
|
||||
|
||||
_REF = "https://example.com/video"
|
||||
|
||||
|
||||
def _asset_held_by_two_jobs(db, status="error"):
|
||||
mine, theirs = User(email="lock-mine@example.com"), User(email="lock-theirs@example.com")
|
||||
db.add_all([mine, theirs])
|
||||
db.flush()
|
||||
asset = MediaAsset(
|
||||
source_kind="url", source_ref=_REF, format_sig="shared", status="error", ref_count=2
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
jobs = [
|
||||
DownloadJob(
|
||||
user_id=owner.id,
|
||||
asset_id=asset.id,
|
||||
source_kind="url",
|
||||
source_ref=_REF,
|
||||
status=status if owner is mine else "error",
|
||||
queue_pos=0,
|
||||
profile_snapshot={},
|
||||
)
|
||||
for owner in (mine, theirs)
|
||||
]
|
||||
db.add_all(jobs)
|
||||
db.commit()
|
||||
return mine, asset, jobs[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "transition", "route", "left"),
|
||||
[
|
||||
("error", service.cancel, cancel_download, "canceled"),
|
||||
("error", service.delete, delete_download, None),
|
||||
("queued", service.cancel, pause_download, "canceled"),
|
||||
],
|
||||
)
|
||||
def test_overlapping_requests_for_one_job_release_its_hold_once(
|
||||
db, status, transition, route, left
|
||||
):
|
||||
user, asset, job = _asset_held_by_two_jobs(db, status)
|
||||
job_id = job.id # read before the delete case removes the row under the expired instance
|
||||
first, second = SessionLocal(), SessionLocal()
|
||||
finished = threading.Event()
|
||||
|
||||
def the_other_request():
|
||||
try:
|
||||
route(job_id, user=user, db=second)
|
||||
except HTTPException:
|
||||
pass # a request after the delete: the job is gone, a 404
|
||||
finally:
|
||||
second.close()
|
||||
finished.set()
|
||||
|
||||
try:
|
||||
locked = _own_job_for_update(first, user, job_id)
|
||||
other = threading.Thread(target=the_other_request)
|
||||
other.start()
|
||||
# The second request waits for the first one's locks instead of deciding on the same row.
|
||||
assert not finished.wait(0.5), "the second request did not wait for the first"
|
||||
transition(first, locked)
|
||||
other.join(10)
|
||||
assert finished.is_set()
|
||||
finally:
|
||||
first.close()
|
||||
|
||||
db.expire_all()
|
||||
mine = select(DownloadJob.status).where(DownloadJob.id == job_id)
|
||||
assert db.execute(mine).scalar_one_or_none() == left
|
||||
assert db.get(MediaAsset, asset.id).ref_count == 1 # the other user's job still holds it
|
||||
|
||||
|
||||
def test_losing_the_asset_insert_race_keeps_the_callers_own_writes(db, monkeypatch):
|
||||
# Retry clears the job's error, THEN may create the cookie asset. When another request inserted that
|
||||
# asset first, the loser used to roll the whole session back — and requeue the job with its stale
|
||||
# error, which a stall or crash later leaves showing under the new failure.
|
||||
user = User(email="race@example.com")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
job = DownloadJob(
|
||||
user_id=user.id,
|
||||
source_kind="url",
|
||||
source_ref=_REF,
|
||||
status="error",
|
||||
error="the old failure",
|
||||
queue_pos=0,
|
||||
profile_snapshot={},
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
job.error = None # the caller's pending write
|
||||
|
||||
real_flush, raced = db.flush, []
|
||||
|
||||
def flush_after_the_other_request_inserted(*args, **kwargs):
|
||||
if not raced: # the other request commits the same asset between our lookup and our insert
|
||||
raced.append(True)
|
||||
with SessionLocal() as other:
|
||||
other.add(
|
||||
MediaAsset(source_kind="url", source_ref=_REF, format_sig="raced", ref_count=0)
|
||||
)
|
||||
other.commit()
|
||||
return real_flush(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(db, "flush", flush_after_the_other_request_inserted)
|
||||
|
||||
asset = service.get_or_create_asset(db, "url", _REF, "raced")
|
||||
db.commit()
|
||||
|
||||
assert raced and asset.format_sig == "raced"
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).error is None
|
||||
@@ -0,0 +1,243 @@
|
||||
"""E11.5 S3 + S4 — the Facebook download path with the user's cookie.
|
||||
|
||||
S3, measured on dev (yt-dlp 2026.07.04, the owner's cookie): reel 806379462330494 fails with the cookie
|
||||
alone and extracts 2560p with `--impersonate chrome`; the reels that already download with the cookie
|
||||
keep their plain first attempt. These pin the attempt order through the real download loop
|
||||
(`pipeline._extract_with_fallback`, with yt-dlp faked at the `YoutubeDL` boundary), and that the
|
||||
LOCKED curl_cffi can actually impersonate — the pair only works inside yt-dlp's own version range, and
|
||||
outside it every target silently reports "unavailable".
|
||||
|
||||
S4, measured on dev and in the owner's Chrome: given a group post's link, yt-dlp finds no formats or
|
||||
downloads another video from the page's feed. The post's own video is the single Video node in the
|
||||
page scripts that carry the post id (prod posts 66, 145, 140, 143, 221; text posts have none while the
|
||||
page holds ~10 other videos), and its watch link downloads.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import yt_dlp
|
||||
from yt_dlp.networking.exceptions import TransportError
|
||||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||||
|
||||
from app.downloads import adapters, pipeline
|
||||
from app.downloads import errors as dlerrors
|
||||
|
||||
_URL = "https://www.facebook.com/reel/806379462330494/?fs=e&fs=e"
|
||||
_POST = "1755988225525547"
|
||||
_GROUP_LINK = f"https://www.facebook.com/groups/983554479435596/permalink/{_POST}/?"
|
||||
_POST_VIDEO = "1408015447857753"
|
||||
_FEED_VIDEO = "1056963063816966" # what yt-dlp downloaded from this post's link: a feed video
|
||||
_CHROME = ImpersonateTarget.from_str("chrome")
|
||||
|
||||
|
||||
def _video_node(video_id: str) -> str:
|
||||
# The measured shape of a post's attachment in the page JSON.
|
||||
return '{"media":{"__typename":"Video","id":"%s","__isNode":"Video"},"styles":{}}' % video_id
|
||||
|
||||
|
||||
def _page(*post_videos: str, post: str = _POST) -> str:
|
||||
attachments = ",".join(_video_node(v) for v in post_videos)
|
||||
return (
|
||||
'<html><script type="application/json" data-sjs>'
|
||||
f'{{"post_id":"{post}","creation_time":1785084554,"attachments":[{attachments}]}}</script>'
|
||||
'<script type="application/json" data-sjs>'
|
||||
f'{{"feed":[{_video_node(_FEED_VIDEO)}]}}</script></html>'
|
||||
)
|
||||
|
||||
|
||||
_WALL = (
|
||||
"ERROR: [facebook] 806379462330494: Cannot parse data; please report this issue on "
|
||||
"https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template."
|
||||
)
|
||||
_NO_FORMATS = "ERROR: [facebook] 2541212983053999: No video formats found!"
|
||||
|
||||
|
||||
def test_the_locked_curl_cffi_can_impersonate_chrome():
|
||||
# YoutubeDL refuses to start when the target is unavailable — the same check a download makes.
|
||||
with yt_dlp.YoutubeDL({"impersonate": ImpersonateTarget.from_str("chrome"), "quiet": True}):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ydl(monkeypatch):
|
||||
"""Fake `yt_dlp.YoutubeDL`: records each attempt's options and plays back `outcomes` in order;
|
||||
`web["page"]` is what a page read returns (`web["fail"]`, when set, is raised instead), and `web`
|
||||
records the pages opened, the byte cap each read asked for, and the links extracted."""
|
||||
seen: list[dict] = []
|
||||
outcomes: list = []
|
||||
web: dict = {"page": "", "fail": None, "opened": [], "reads": [], "extracted": []}
|
||||
|
||||
class FakeYDL:
|
||||
def __init__(self, opts):
|
||||
seen.append(opts)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def urlopen(self, url):
|
||||
web["opened"].append(url)
|
||||
if web["fail"] is not None:
|
||||
raise web["fail"]
|
||||
body = web["page"].encode()
|
||||
|
||||
class Response:
|
||||
def read(self, amt=None):
|
||||
web["reads"].append(amt)
|
||||
return body
|
||||
|
||||
return Response()
|
||||
|
||||
def extract_info(self, url, download):
|
||||
web["extracted"].append(url)
|
||||
outcome = outcomes.pop(0)
|
||||
if isinstance(outcome, str):
|
||||
raise yt_dlp.utils.DownloadError(outcome)
|
||||
return outcome
|
||||
|
||||
monkeypatch.setattr(pipeline.yt_dlp, "YoutubeDL", FakeYDL)
|
||||
monkeypatch.setattr(pipeline, "_touch_job", lambda job_id: None)
|
||||
monkeypatch.setattr(pipeline, "_set_job", lambda job_id, **fields: None)
|
||||
return seen, outcomes, web
|
||||
|
||||
|
||||
def _run(tmp_path, cookiefile, url=_URL):
|
||||
return pipeline._extract_with_fallback(
|
||||
1, 1, url, {}, tmp_path, adapters.FacebookAdapter(), cookiefile=cookiefile
|
||||
)
|
||||
|
||||
|
||||
def test_a_cookie_download_at_the_wall_is_retried_as_chrome(ydl, tmp_path):
|
||||
seen, outcomes, web = ydl
|
||||
outcomes += [_WALL, {"id": "806379462330494"}]
|
||||
|
||||
assert _run(tmp_path, "/cookies.txt") == {"id": "806379462330494"}
|
||||
|
||||
assert [o.get("impersonate") for o in seen] == [None, _CHROME]
|
||||
assert all(o["cookiefile"] == "/cookies.txt" for o in seen) # the retry stays signed in
|
||||
assert web["opened"] == [] # a reel is not a group post: no page read
|
||||
|
||||
|
||||
def test_without_a_cookie_the_wall_is_the_login_wall_and_final(ydl, tmp_path):
|
||||
seen, outcomes, _ = ydl
|
||||
outcomes += [_WALL]
|
||||
|
||||
with pytest.raises(yt_dlp.utils.DownloadError):
|
||||
_run(tmp_path, None)
|
||||
|
||||
assert len(seen) == 1 and "impersonate" not in seen[0]
|
||||
|
||||
|
||||
def test_other_facebook_failures_are_not_retried_as_chrome(ydl, tmp_path):
|
||||
seen, outcomes, _ = ydl
|
||||
outcomes += [_NO_FORMATS]
|
||||
|
||||
with pytest.raises(yt_dlp.utils.DownloadError):
|
||||
_run(tmp_path, "/cookies.txt")
|
||||
|
||||
assert len(seen) == 1 and "impersonate" not in seen[0]
|
||||
|
||||
|
||||
# --- S4: a group post downloads ITS video, or fails saying it has none ----------------------------
|
||||
|
||||
|
||||
def test_a_cookie_group_post_downloads_the_posts_own_video(ydl, tmp_path):
|
||||
seen, outcomes, web = ydl
|
||||
web["page"] = _page(_POST_VIDEO)
|
||||
outcomes += [_WALL, {"id": _POST_VIDEO}] # the watch link needs Chrome too (measured)
|
||||
|
||||
assert _run(tmp_path, "/cookies.txt", _GROUP_LINK) == {"id": _POST_VIDEO}
|
||||
|
||||
assert web["opened"] == [_GROUP_LINK]
|
||||
# Never an unbounded read.
|
||||
assert web["reads"] == [adapters.FacebookAdapter._POST_PAGE_MAX_BYTES]
|
||||
assert seen[0]["cookiefile"] == "/cookies.txt" # the post page is read signed in
|
||||
watch = f"https://www.facebook.com/watch/?v={_POST_VIDEO}"
|
||||
# Never the post link: that is where the feed video comes from.
|
||||
assert web["extracted"] == [watch, watch]
|
||||
assert [o.get("impersonate") for o in seen[1:]] == [None, _CHROME]
|
||||
|
||||
|
||||
def test_a_group_post_without_a_single_video_fails_saying_so(ydl, tmp_path):
|
||||
_, _, web = ydl
|
||||
web["page"] = _page() # a text post: the only video on the page is the feed's
|
||||
|
||||
with pytest.raises(RuntimeError) as exc:
|
||||
_run(tmp_path, "/cookies.txt", _GROUP_LINK)
|
||||
|
||||
assert web["extracted"] == [] # yt-dlp is not left to guess from the post link
|
||||
assert dlerrors.diagnose(str(exc.value), url=_GROUP_LINK, cookie_attached=True) == (
|
||||
"format_unavailable",
|
||||
"facebook.group_post_video",
|
||||
)
|
||||
|
||||
|
||||
def test_a_group_post_page_without_the_post_is_the_refused_cookie(ydl, tmp_path):
|
||||
# Measured with a refused cookie (E11.5 review): Facebook answers the post link with its login page —
|
||||
# a login form, no post id anywhere. Saying "this post has no video" would blame the post.
|
||||
_, _, web = ydl
|
||||
web["page"] = '<html><title>ChatGPT magyarul</title><form id="login_form"></form></html>'
|
||||
|
||||
with pytest.raises(RuntimeError) as exc:
|
||||
_run(tmp_path, "/cookies.txt", _GROUP_LINK)
|
||||
|
||||
assert web["extracted"] == []
|
||||
assert dlerrors.diagnose(str(exc.value), url=_GROUP_LINK, cookie_attached=True) == (
|
||||
"login_required",
|
||||
"facebook.cookie_rejected",
|
||||
)
|
||||
|
||||
|
||||
def test_a_failed_post_page_read_fails_the_download_as_itself(ydl, tmp_path):
|
||||
# A read that failed says nothing about the post: it must not turn into "no video in this post".
|
||||
_, _, web = ydl
|
||||
web["fail"] = TransportError("Connection reset by peer")
|
||||
|
||||
with pytest.raises(TransportError) as exc:
|
||||
_run(tmp_path, "/cookies.txt", _GROUP_LINK)
|
||||
|
||||
assert web["extracted"] == []
|
||||
assert dlerrors.diagnose(str(exc.value), url=_GROUP_LINK, cookie_attached=True) == (
|
||||
"network",
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def test_without_a_cookie_the_group_post_link_goes_to_ytdlp_unchanged(ydl, tmp_path):
|
||||
_, outcomes, web = ydl
|
||||
outcomes += [_NO_FORMATS]
|
||||
|
||||
with pytest.raises(yt_dlp.utils.DownloadError):
|
||||
_run(tmp_path, None, _GROUP_LINK)
|
||||
|
||||
assert web["opened"] == [] # signed out the page is the login wall: nothing to read
|
||||
assert web["extracted"] == [_GROUP_LINK]
|
||||
|
||||
|
||||
def test_the_group_post_link_forms_are_recognised():
|
||||
for url in (
|
||||
_GROUP_LINK,
|
||||
f"https://www.facebook.com/groups/983554479435596/posts/{_POST}/",
|
||||
f"https://m.facebook.com/groups/983554479435596/permalink/{_POST}/?",
|
||||
f"https://www.facebook.com/groups/chatgptmagyarul/posts/{_POST}",
|
||||
):
|
||||
assert adapters.facebook_group_post_id(url) == _POST, url
|
||||
for url in (
|
||||
_URL,
|
||||
f"https://www.facebook.com/watch/?v={_POST_VIDEO}",
|
||||
f"https://facebook.com.evil/groups/x/permalink/{_POST}/",
|
||||
"https://www.facebook.com/groups/983554479435596/",
|
||||
):
|
||||
assert adapters.facebook_group_post_id(url) is None, url
|
||||
assert adapters.SourceAdapter().resolve_url(_GROUP_LINK, {"cookiefile": "/c"}) == _GROUP_LINK
|
||||
|
||||
|
||||
def test_only_the_single_video_of_the_posts_own_blocks_counts():
|
||||
assert adapters.facebook_post_video_id(_page(_POST_VIDEO), _POST) == _POST_VIDEO
|
||||
# No video in the post, although the page has one elsewhere.
|
||||
assert adapters.facebook_post_video_id(_page(), _POST) is None
|
||||
# Two videos: guessing between them would be the wrong-video bug again.
|
||||
assert adapters.facebook_post_video_id(_page(_POST_VIDEO, "2063302677911049"), _POST) is None
|
||||
# A longer number that merely STARTS with the post id is another object.
|
||||
assert adapters.facebook_post_video_id(_page(_POST_VIDEO, post=_POST + "0"), _POST) is None
|
||||
@@ -0,0 +1,63 @@
|
||||
"""The scheduled RSS poll and recent backfill cover FOLLOWED channels only (owner decision, 2026-09-17).
|
||||
|
||||
A channel row also comes from a search result, a playlist item or a subscription somebody dropped,
|
||||
and nothing deletes those rows. Polling all of them was 582 of 979 channels on prod with no follower,
|
||||
three of them dead feeds answering 404 every 20 minutes. Followed = subscribed by someone, or explored
|
||||
by someone. The admin card counting "channels to sync" must use the same selection, or it counts
|
||||
channels the backfill will never touch. DB lane.
|
||||
"""
|
||||
|
||||
from app.models import Channel, ExploredChannel, Subscription
|
||||
from app.sync import runner
|
||||
|
||||
|
||||
def _channels(db, make_user):
|
||||
user = make_user()
|
||||
for cid in ("UC_sub", "UC_explored", "UC_search", "UC_playlist"):
|
||||
db.add(Channel(id=cid, title=cid, from_search=cid == "UC_search"))
|
||||
db.flush()
|
||||
db.add(Subscription(user_id=user.id, channel_id="UC_sub"))
|
||||
db.add(ExploredChannel(user_id=user.id, channel_id="UC_explored"))
|
||||
db.commit()
|
||||
return user
|
||||
|
||||
|
||||
def test_the_rss_poll_skips_channels_nobody_follows(db, make_user, monkeypatch):
|
||||
_channels(db, make_user)
|
||||
polled: list[str] = []
|
||||
monkeypatch.setattr(runner, "fetch_channel_feed", lambda cid: polled.append(cid) or [])
|
||||
monkeypatch.setattr(runner, "apply_rss_feed", lambda db, ch, entries: 0)
|
||||
assert runner.run_rss_poll(db) == {"new": 0, "failed": 0}
|
||||
assert sorted(polled) == ["UC_explored", "UC_sub"]
|
||||
|
||||
|
||||
def test_the_recent_backfill_skips_channels_nobody_follows(db, make_user, monkeypatch):
|
||||
user = _channels(db, make_user)
|
||||
done: list[str] = []
|
||||
|
||||
class _Client:
|
||||
def __init__(self, *a):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(runner, "get_service_user", lambda db: user)
|
||||
monkeypatch.setattr(runner, "YouTubeClient", _Client)
|
||||
monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
runner, "backfill_channel_recent", lambda db, yt, ch: done.append(ch.id) or 0
|
||||
)
|
||||
monkeypatch.setattr(runner, "reconcile_full_history", lambda db: 0)
|
||||
runner.run_recent_backfill(db)
|
||||
assert sorted(done) == ["UC_explored", "UC_sub"]
|
||||
|
||||
|
||||
def test_the_dashboard_counts_the_channels_the_backfill_will_process(db, make_user, as_user):
|
||||
_channels(db, make_user)
|
||||
admin = make_user(role="admin")
|
||||
queue = as_user(admin).get("/api/admin/scheduler").json()["queue"]
|
||||
assert queue["channels_recent_pending"] == 2
|
||||
@@ -158,3 +158,24 @@ def test_a_normal_cut_list_is_untouched():
|
||||
segs = [{"start_s": 0, "end_s": 5}, {"start_s": 10, "end_s": 12}]
|
||||
out = editmod.normalize_edit_spec({"segments": segs})
|
||||
assert len(out["segments"]) == 2
|
||||
|
||||
|
||||
def test_audio_versions_its_cache_identity_apart_from_video(monkeypatch):
|
||||
# The .mp3/.m4a-stays-itself fix must re-derive cached remuxed audio WITHOUT re-downloading every
|
||||
# cached video: bumping one version moves only its own mode's signature.
|
||||
audio, video = {"mode": "a", "audio_format": "mp3"}, {"mode": "av"}
|
||||
before = formats.format_sig(audio), formats.format_sig(video)
|
||||
monkeypatch.setattr(formats, "_AUDIO_SIG_VERSION", formats._AUDIO_SIG_VERSION + 1)
|
||||
assert formats.format_sig(audio) != before[0]
|
||||
assert formats.format_sig(video) == before[1]
|
||||
monkeypatch.setattr(formats, "_SIG_VERSION", formats._SIG_VERSION + 1)
|
||||
assert formats.format_sig(video) != before[1]
|
||||
|
||||
|
||||
def test_the_mp3_fix_moved_the_audio_cache_identity(monkeypatch):
|
||||
# Before the fix every mode hashed with version 4, so an audio signature still at 4 would find the
|
||||
# cached mp3-remuxed-into-mp4 assets iPad refuses (prod job 268) instead of re-deriving them.
|
||||
audio = {"mode": "a", "audio_format": "mp3"}
|
||||
current = formats.format_sig(audio)
|
||||
monkeypatch.setattr(formats, "_AUDIO_SIG_VERSION", 4)
|
||||
assert formats.format_sig(audio) != current
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Idle HLS sessions are reaped on a timer, and a viewer who is merely PAUSED keeps theirs.
|
||||
|
||||
plex/stream.py promises "a periodic reaper kills idle ones", but `reap_idle` ran only inside
|
||||
POST /session: after the last playback, an abandoned session (its ffmpeg still working through the
|
||||
film, its directory a whole segmented copy of it) lived until somebody pressed play again
|
||||
(deep-review round 4).
|
||||
|
||||
And it must not take a PAUSED viewer's session: a remux finishes long before the film does, and a
|
||||
paused player with a full buffer fetches nothing, so "finished and unfetched for 30 s" does not mean
|
||||
"abandoned". Round 4 bought that safety by keeping EVERY finished session for the full ten-minute
|
||||
idle window — one segmented copy of a film per item, on the volume the download center shares. Round
|
||||
5 replaced the blanket retention with the signal that already existed: an open player checkpoints its
|
||||
position every 10 s whether or not it is playing, and that route says "still here".
|
||||
|
||||
Both call sites are pinned here, because the bug each time was that only one of them moved: the
|
||||
periodic pass, and the fact that starting a session no longer sweeps OTHER viewers' sessions at all.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.main as app_main
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import PlexItem, PlexLibrary, User
|
||||
from app.plex import paths as plex_paths
|
||||
from app.plex import stream
|
||||
|
||||
|
||||
class _Configured:
|
||||
def __enter__(self):
|
||||
return None
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
|
||||
def test_the_periodic_pass_is_the_one_that_runs_and_it_uses_the_short_rule(monkeypatch):
|
||||
async def _no_sweep():
|
||||
return None
|
||||
|
||||
reaped: list[int] = []
|
||||
monkeypatch.setattr(app_main, "SessionLocal", _Configured)
|
||||
monkeypatch.setattr(app_main.state, "is_configured", lambda db: True)
|
||||
monkeypatch.setattr(app_main, "_sweep_hls_orphans", _no_sweep)
|
||||
monkeypatch.setattr(app_main, "start_scheduler", lambda: None)
|
||||
monkeypatch.setattr(app_main, "shutdown_scheduler", lambda: None)
|
||||
monkeypatch.setattr(app_main.plex_subtitles, "shutdown", lambda: None)
|
||||
monkeypatch.setattr(app_main, "_HLS_REAP_EVERY_S", 0.01)
|
||||
# `reap_idle`, i.e. the SHORT finished rule: a finished session nobody has said they still want
|
||||
# is a whole copy of a film sitting on the download volume.
|
||||
monkeypatch.setattr(app_main.plex_stream, "reap_idle", lambda: reaped.append(1) or 0)
|
||||
|
||||
async def _cycle():
|
||||
async with app_main.lifespan(app):
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
asyncio.run(_cycle())
|
||||
assert len(reaped) >= 2, "no periodic reap ran while the app was up"
|
||||
|
||||
|
||||
class _Proc:
|
||||
def __init__(self, finished: bool):
|
||||
self.finished = finished
|
||||
self.killed = False
|
||||
|
||||
def poll(self):
|
||||
return 0 if self.finished else None
|
||||
|
||||
def terminate(self):
|
||||
self.killed = True
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self, tmp_path, name: str, finished: bool, idle_s: float):
|
||||
self.dir = tmp_path / name
|
||||
self.dir.mkdir()
|
||||
self.proc = _Proc(finished)
|
||||
self.last_access = time.time() - idle_s
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sessions(monkeypatch):
|
||||
live: dict = {}
|
||||
monkeypatch.setattr(stream, "_sessions", live)
|
||||
return live
|
||||
|
||||
|
||||
def test_the_periodic_pass_reaps_a_finished_session_nobody_has_claimed(tmp_path, sessions):
|
||||
sessions["12345"] = _Session(tmp_path, "done", finished=True, idle_s=45)
|
||||
assert stream.reap_idle() == 1
|
||||
assert sessions == {}
|
||||
|
||||
|
||||
def test_the_periodic_pass_still_reaps_what_nobody_has_touched_for_the_idle_limit(
|
||||
tmp_path, sessions
|
||||
):
|
||||
sessions["1"] = _Session(
|
||||
tmp_path, "gone_done", finished=True, idle_s=stream._SESSION_IDLE_S + 5
|
||||
)
|
||||
sessions["2"] = _Session(
|
||||
tmp_path, "gone_live", finished=False, idle_s=stream._SESSION_IDLE_S + 5
|
||||
)
|
||||
assert stream.reap_idle() == 2
|
||||
assert sessions == {}
|
||||
|
||||
|
||||
def test_touching_a_session_keeps_it_through_the_short_rule(tmp_path, sessions):
|
||||
s = _Session(tmp_path, "paused", finished=True, idle_s=90)
|
||||
sessions["12345"] = s
|
||||
stream.touch_session("12345")
|
||||
assert stream.reap_idle() == 0
|
||||
assert "12345" in sessions and s.dir.exists() and not s.proc.killed
|
||||
|
||||
|
||||
# --- the two call sites, driven through the real routes (DB lane) -------------------------------
|
||||
|
||||
|
||||
def _library_item(db, rating_key: str, *, playable: str = "remux"):
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
it = PlexItem(
|
||||
rating_key=rating_key,
|
||||
library_id=lib.id,
|
||||
kind="movie",
|
||||
title="A Movie",
|
||||
file_path="/media/a.mkv",
|
||||
playable=playable,
|
||||
)
|
||||
db.add(it)
|
||||
db.commit()
|
||||
return it
|
||||
|
||||
|
||||
def _client(db, user):
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_a_paused_viewers_checkpoint_keeps_their_session_alive(db, monkeypatch, tmp_path, sessions):
|
||||
"""The whole reason the short rule is safe again: a paused player fetches nothing, so
|
||||
`last_access` never moved on its own — but it does keep checkpointing its position."""
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
it = _library_item(db, "paused1")
|
||||
user = User(email="paused@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
s = _Session(tmp_path, "paused", finished=True, idle_s=90)
|
||||
sessions["paused1"] = s
|
||||
try:
|
||||
resp = _client(db, user).post(
|
||||
f"/api/plex/item/{it.rating_key}/progress",
|
||||
json={"position_seconds": 600, "duration_seconds": 3000, "final": False},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert stream.reap_idle() == 0, "the checkpoint did not say the viewer is still there"
|
||||
assert "paused1" in sessions and not s.proc.killed
|
||||
|
||||
|
||||
def test_starting_a_session_leaves_another_viewers_paused_one_alone(
|
||||
db, monkeypatch, tmp_path, sessions
|
||||
):
|
||||
"""The opportunistic sweep this route used to run was not scoped to the key being started: it
|
||||
walked the WHOLE registry with the 30 s finished rule, so pressing play on any item killed a
|
||||
paused viewer's session — the very case round 4 set out to protect (found in round 5)."""
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
monkeypatch.setattr(plex_paths, "local_media_path", lambda db_, fp: "/local/a.mkv")
|
||||
started = _library_item(db, "starting")
|
||||
user = User(email="starter@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
paused = _Session(tmp_path, "paused", finished=True, idle_s=90)
|
||||
sessions["someone-else"] = paused
|
||||
|
||||
def fake_start(db_, item, start, audio=None, aoff=0.0, multi=False, transcode=False):
|
||||
return SimpleNamespace(
|
||||
id="sid",
|
||||
media_start_s=start,
|
||||
entry="index.m3u8",
|
||||
transcode=transcode,
|
||||
hw=False,
|
||||
tonemap=False,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(stream, "start_session", fake_start)
|
||||
try:
|
||||
resp = _client(db, user).post(f"/api/plex/stream/{started.rating_key}/session")
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert "someone-else" in sessions and not paused.proc.killed
|
||||
@@ -0,0 +1,97 @@
|
||||
"""`users.last_login_at` (E22 S1), DB-lane.
|
||||
|
||||
The roadmap assumed stamping it in `_establish_session` would cover Google and password alike. It
|
||||
would not have: only password login called it, and the Google callback carried its own copy of the
|
||||
same four lines. So these pin BOTH doors through the real routes — plus the two that must not stamp
|
||||
(a refused password, the shared demo account) and the admin list that shows it.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from app.config import settings
|
||||
from app.models import DemoWhitelist
|
||||
from app.routes import auth as auth_routes
|
||||
|
||||
_PW = "correct horse battery"
|
||||
|
||||
|
||||
def _recent(ts: datetime | None) -> bool:
|
||||
return ts is not None and datetime.now(timezone.utc) - ts < timedelta(minutes=1)
|
||||
|
||||
|
||||
def test_a_password_sign_in_stamps_last_login(client, db, make_user):
|
||||
user = make_user(email="pw@example.com", password=_PW)
|
||||
assert user.last_login_at is None
|
||||
|
||||
r = client.post("/auth/password-login", json={"email": "pw@example.com", "password": _PW})
|
||||
|
||||
assert r.status_code == 200
|
||||
db.refresh(user)
|
||||
assert _recent(user.last_login_at)
|
||||
|
||||
|
||||
def test_a_refused_password_does_not_stamp(client, db, make_user):
|
||||
user = make_user(email="wrong@example.com", password=_PW)
|
||||
|
||||
r = client.post(
|
||||
"/auth/password-login", json={"email": "wrong@example.com", "password": "not it"}
|
||||
)
|
||||
|
||||
assert r.status_code == 401
|
||||
db.refresh(user)
|
||||
assert user.last_login_at is None
|
||||
|
||||
|
||||
class _FakeGoogle:
|
||||
"""authlib's client, reduced to the one call the callback makes after Google redirects back."""
|
||||
|
||||
def __init__(self, userinfo: dict):
|
||||
self._userinfo = userinfo
|
||||
|
||||
async def authorize_access_token(self, _request):
|
||||
return {"userinfo": self._userinfo, "access_token": "at", "scope": "openid email profile"}
|
||||
|
||||
|
||||
def test_a_google_sign_in_stamps_last_login(client, db, make_user, monkeypatch):
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
from app import security
|
||||
|
||||
# The callback stores the OAuth grant encrypted; a throwaway key stands in for TOKEN_ENCRYPTION_KEY.
|
||||
monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key()))
|
||||
user = make_user(email="g@example.com", password=None, google_sub="sub-g")
|
||||
monkeypatch.setattr(settings, "admin_emails", "g@example.com") # the bootstrap allow-list
|
||||
fake = _FakeGoogle({"sub": "sub-g", "email": "g@example.com", "email_verified": True})
|
||||
monkeypatch.setattr(auth_routes, "google_oauth", lambda _db: fake)
|
||||
|
||||
r = client.get("/auth/callback", follow_redirects=False)
|
||||
|
||||
assert r.status_code in (302, 307) and r.headers["location"] == "/"
|
||||
db.refresh(user)
|
||||
assert _recent(user.last_login_at)
|
||||
# …and it is a real sign-in, not only a stamp: the session now answers as this account.
|
||||
assert client.get("/api/me").json()["id"] == user.id
|
||||
|
||||
|
||||
def test_the_demo_entry_does_not_stamp_the_shared_account(client, db):
|
||||
db.add(DemoWhitelist(email="visitor@example.com"))
|
||||
db.commit()
|
||||
|
||||
r = client.post("/auth/demo", json={"email": "visitor@example.com"})
|
||||
|
||||
assert r.json() == {"authenticated": True}
|
||||
demo = auth_routes.get_or_create_demo_user(db)
|
||||
db.refresh(demo)
|
||||
assert demo.last_login_at is None
|
||||
|
||||
|
||||
def test_the_admin_user_list_carries_last_login(as_user, db, make_user):
|
||||
admin = make_user(email="boss@example.com", role="admin")
|
||||
never = make_user(email="never@example.com")
|
||||
admin.last_login_at = datetime(2026, 9, 16, 8, 30, tzinfo=timezone.utc)
|
||||
db.commit()
|
||||
|
||||
rows = {u["email"]: u for u in as_user(admin).get("/api/admin/users").json()}
|
||||
|
||||
assert rows["boss@example.com"]["last_login_at"].startswith("2026-09-16T08:30")
|
||||
assert rows[never.email]["last_login_at"] is None
|
||||
@@ -0,0 +1,320 @@
|
||||
"""E22 S2 — the one logging configuration, its redaction, and the request line that replaced
|
||||
uvicorn's access log.
|
||||
|
||||
Redaction is the part worth guarding hardest: it is applied at the HANDLER, so these cases are the
|
||||
only thing standing between a future call site and a token in a log we attach to a Gitea issue.
|
||||
The request-line cases pin what uvicorn's access log got wrong — the query string (`?account=1`,
|
||||
`?token=…`) was printed on every request, and nothing correlated two lines of the same request.
|
||||
|
||||
Pure logic + an in-process ASGI app: no database.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app import logs
|
||||
from app.config import LogLevel, choices_of
|
||||
|
||||
|
||||
class _Sink(logging.Handler):
|
||||
"""Captures records AND their formatted JSON, with the same filter/formatter the real stdout
|
||||
handler uses — so a test sees exactly the bytes a deployment would write."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(level=logging.NOTSET)
|
||||
self.lines: list[str] = []
|
||||
self.records: list[logging.LogRecord] = []
|
||||
self.setFormatter(logs.JsonFormatter())
|
||||
self.addFilter(logs.ContextFilter())
|
||||
|
||||
def emit(self, record):
|
||||
self.lines.append(self.format(record))
|
||||
self.records.append(record)
|
||||
|
||||
def objects(self) -> list[dict]:
|
||||
return [json.loads(line) for line in self.lines]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sink():
|
||||
"""Attached to `siftlode` (not root) so a `configure_logging` call inside a test — which
|
||||
replaces the ROOT handlers — can't silently detach it mid-test."""
|
||||
logger = logging.getLogger("siftlode")
|
||||
handler = _Sink()
|
||||
previous = logger.level
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logs.TRACE)
|
||||
try:
|
||||
yield handler
|
||||
finally:
|
||||
logger.removeHandler(handler)
|
||||
logger.setLevel(previous)
|
||||
|
||||
|
||||
# --- redaction ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, must_not_contain",
|
||||
[
|
||||
("user signed in: someone@example.com", "someone@example.com"),
|
||||
("callback https://app/auth/callback?code=4/0AY&state=xyz", "4/0AY"),
|
||||
("mail link https://app/#verify=abc123def", "abc123def"),
|
||||
("share https://app/s?token=SHARETOKEN", "SHARETOKEN"),
|
||||
("reset https://app/#reset=RESETTOKEN", "RESETTOKEN"),
|
||||
],
|
||||
)
|
||||
def test_free_text_secrets_never_reach_the_line(text, must_not_contain):
|
||||
assert must_not_contain not in logs.redact_text(text)
|
||||
|
||||
|
||||
def test_a_key_that_names_a_secret_is_dropped_whole():
|
||||
# Not truncated to a prefix: a prefix is still a leak and is useless for triage anyway.
|
||||
out = logs.redact(
|
||||
{
|
||||
"password": "hunter2",
|
||||
"current_password": "hunter2",
|
||||
"plex_server_token": "abc",
|
||||
"Cookie": "session=xyz",
|
||||
"authorization": "Bearer abc",
|
||||
"nested": {"refresh_token": "rt", "keep": "visible"},
|
||||
"list": [{"api_key": "k"}],
|
||||
"count": 7,
|
||||
}
|
||||
)
|
||||
assert out["password"] == logs.REDACTED
|
||||
assert out["current_password"] == logs.REDACTED
|
||||
assert out["plex_server_token"] == logs.REDACTED
|
||||
assert out["Cookie"] == logs.REDACTED
|
||||
assert out["authorization"] == logs.REDACTED
|
||||
assert out["nested"]["refresh_token"] == logs.REDACTED
|
||||
assert out["nested"]["keep"] == "visible" # only the named keys go
|
||||
assert out["list"][0]["api_key"] == logs.REDACTED
|
||||
assert out["count"] == 7
|
||||
|
||||
|
||||
def test_an_email_in_an_attribute_value_is_scrubbed_too():
|
||||
assert logs.redact({"to": "a.b+tag@mail.example.org"})["to"] == logs.REDACTED_EMAIL
|
||||
|
||||
|
||||
# The SAME cases frontend/src/lib/log.contract.test.ts runs against the browser's copy of redact():
|
||||
# the one thing that keeps the two logs' idea of "safe to keep" from drifting apart.
|
||||
_REDACTION_CASES = json.loads(
|
||||
(Path(__file__).parent / "fixtures" / "log_redaction_cases.json").read_text(encoding="utf-8")
|
||||
)["cases"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", _REDACTION_CASES, ids=[c["name"] for c in _REDACTION_CASES])
|
||||
def test_the_shared_redaction_cases(case):
|
||||
assert logs.redact(case["value"], key=case["key"]) == case["expected"]
|
||||
|
||||
|
||||
def test_redaction_runs_on_the_message_the_attrs_and_the_traceback(sink):
|
||||
log = logging.getLogger("siftlode.test")
|
||||
try:
|
||||
raise RuntimeError("boom for someone@example.com")
|
||||
except RuntimeError:
|
||||
log.exception(
|
||||
"failed for %s",
|
||||
"someone@example.com",
|
||||
extra=logs.event("test.failed", password="hunter2", url="https://x/?token=T0K3N"),
|
||||
)
|
||||
line = sink.lines[-1]
|
||||
assert "someone@example.com" not in line
|
||||
assert "hunter2" not in line
|
||||
assert "T0K3N" not in line
|
||||
payload = json.loads(line)
|
||||
assert payload["attrs"]["password"] == logs.REDACTED
|
||||
assert "RuntimeError" in payload["exc"] # the diagnosis survives the scrubbing
|
||||
|
||||
|
||||
# --- event shape ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_every_line_carries_the_agreed_fields(sink):
|
||||
logging.getLogger("siftlode.test").info("hello", extra=logs.event("test.hello", n=1))
|
||||
payload = sink.objects()[-1]
|
||||
for field in ("ts", "level", "sev", "source", "logger", "msg", "event", "version"):
|
||||
assert field in payload, field
|
||||
assert payload["level"] == "INFO"
|
||||
assert payload["sev"] == 9
|
||||
assert payload["event"] == "test.hello"
|
||||
assert payload["attrs"] == {"n": 1}
|
||||
|
||||
|
||||
def test_a_plain_log_call_still_produces_a_well_formed_line(sink):
|
||||
"""An event code is optional — 125 existing call sites have none yet (E22.5 adds them), and they
|
||||
must not be worse off than before this module."""
|
||||
logging.getLogger("siftlode.test").warning("just a message")
|
||||
payload = sink.objects()[-1]
|
||||
assert payload["msg"] == "just a message"
|
||||
assert payload["level"] == "WARNING"
|
||||
assert "event" not in payload
|
||||
|
||||
|
||||
def test_a_broken_format_call_reports_itself_instead_of_raising():
|
||||
"""A record is formatted directly rather than logged: a genuinely broken call ALSO breaks
|
||||
pytest's own capture handler (stdlib formatting raises there too), which would fail the test for
|
||||
the wrong reason. What is under test is our formatter's refusal to raise."""
|
||||
record = logging.LogRecord(
|
||||
"siftlode.test", logging.INFO, __file__, 0, "two args %s %s", ("only-one",), None
|
||||
)
|
||||
payload = json.loads(logs.JsonFormatter().format(record))
|
||||
assert "log formatting failed" in payload["msg"]
|
||||
|
||||
|
||||
def test_bound_correlation_appears_and_is_released(sink):
|
||||
log = logging.getLogger("siftlode.test")
|
||||
with logs.bind(job_id=42):
|
||||
log.info("inside")
|
||||
log.info("outside")
|
||||
assert sink.objects()[-2]["job_id"] == 42
|
||||
assert "job_id" not in sink.objects()[-1]
|
||||
|
||||
|
||||
def test_every_selectable_level_resolves_to_a_real_logging_level():
|
||||
# The picker's options come from `LogLevel`; each must be a level `logging` knows, TRACE included.
|
||||
assert [logs.resolve_level(n) for n in choices_of(LogLevel)] == [
|
||||
logs.TRACE,
|
||||
logging.DEBUG,
|
||||
logging.INFO,
|
||||
logging.WARNING,
|
||||
logging.ERROR,
|
||||
]
|
||||
with pytest.raises(ValueError):
|
||||
logs.resolve_level("nonsense")
|
||||
|
||||
|
||||
def test_configure_logging_silences_the_uvicorn_access_log_and_keeps_ours_propagating():
|
||||
logs.configure_logging("api")
|
||||
assert logging.getLogger("uvicorn.access").getEffectiveLevel() >= logging.CRITICAL
|
||||
# propagate stays on so every line reaches the ONE root handler — and so pytest's caplog works.
|
||||
assert logging.getLogger("siftlode.anything").propagate is True
|
||||
assert len(logging.getLogger().handlers) == 1
|
||||
|
||||
|
||||
# --- the request line -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def _client() -> TestClient:
|
||||
app = FastAPI()
|
||||
app.middleware("http")(logs.log_requests)
|
||||
|
||||
@app.get("/api/videos/{video_id}")
|
||||
def _video(video_id: str, q: str | None = None):
|
||||
return {"ok": video_id}
|
||||
|
||||
@app.get("/healthz")
|
||||
def _health():
|
||||
return {"ok": True}
|
||||
|
||||
@app.get("/boom")
|
||||
def _boom():
|
||||
raise RuntimeError("kaboom")
|
||||
|
||||
return TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
def test_the_request_line_has_the_route_template_and_no_query_string(sink):
|
||||
logging.getLogger("siftlode.http").setLevel(logs.TRACE)
|
||||
r = _client().get("/api/videos/abc123?q=cats&token=SECRET")
|
||||
assert r.status_code == 200
|
||||
payload = sink.objects()[-1]
|
||||
assert payload["event"] == "http.request"
|
||||
# The TEMPLATE, so every video request groups into one queryable path...
|
||||
assert payload["attrs"]["path"] == "/api/videos/{video_id}"
|
||||
# ...and the query string — uvicorn's access log printed it, `?account=1` and all — is gone.
|
||||
assert "SECRET" not in sink.lines[-1]
|
||||
assert "cats" not in sink.lines[-1]
|
||||
assert payload["attrs"]["status"] == 200
|
||||
assert isinstance(payload["attrs"]["duration_ms"], float)
|
||||
|
||||
|
||||
def test_anonymous_request_lines_are_capped_per_client_for_storage_but_still_logged(sink):
|
||||
# A scanner's 404s are WARNINGs, and WARNINGs are stored. Past the cap they stay on stdout but are
|
||||
# marked not-for-storage; a signed-in request is never capped. No address appears in any line.
|
||||
logs._anonymous_lines.reset()
|
||||
client = _client()
|
||||
for _ in range(logs.ANONYMOUS_LINES_PER_MINUTE + 3):
|
||||
client.get("/wp-login.php")
|
||||
lines = sink.objects()[-(logs.ANONYMOUS_LINES_PER_MINUTE + 3) :]
|
||||
assert len(lines) == logs.ANONYMOUS_LINES_PER_MINUTE + 3 # stdout keeps every one
|
||||
records = sink.records[-(logs.ANONYMOUS_LINES_PER_MINUTE + 3) :]
|
||||
stored = [getattr(r, logs.PERSIST, True) for r in records]
|
||||
assert stored.count(False) == 3 and all(stored[: logs.ANONYMOUS_LINES_PER_MINUTE])
|
||||
assert "testclient" not in "".join(sink.lines) # the client key never reaches the line
|
||||
logs._anonymous_lines.reset()
|
||||
|
||||
|
||||
def test_a_signed_in_request_is_never_capped_and_names_its_account(sink):
|
||||
logs._anonymous_lines.reset()
|
||||
app = FastAPI()
|
||||
app.middleware("http")(logs.log_requests)
|
||||
|
||||
@app.get("/api/me")
|
||||
def _me(request: Request):
|
||||
request.state.user_id = 42 # what the auth dependency does
|
||||
return {}
|
||||
|
||||
client = TestClient(app)
|
||||
for _ in range(logs.ANONYMOUS_LINES_PER_MINUTE + 3):
|
||||
client.get("/api/me")
|
||||
records = sink.records[-(logs.ANONYMOUS_LINES_PER_MINUTE + 3) :]
|
||||
assert all(getattr(r, logs.PERSIST, True) for r in records)
|
||||
assert sink.objects()[-1]["user_id"] == 42 and "user_id" not in sink.objects()[-1]["attrs"]
|
||||
|
||||
|
||||
def test_the_request_id_is_echoed_and_shared_by_every_line_of_that_request(sink):
|
||||
client = _client()
|
||||
r = client.get("/api/videos/abc123")
|
||||
rid = r.headers["X-Request-Id"]
|
||||
assert rid and sink.objects()[-1]["request_id"] == rid
|
||||
|
||||
|
||||
def test_the_route_template_goes_back_to_the_browser_but_never_a_raw_path(sink):
|
||||
# The browser logs its own copy of each request, grouped by this template (frontend lib/http.ts).
|
||||
client = _client()
|
||||
assert client.get("/api/videos/SeCrEt-VaLuE").headers["X-Route"] == "/api/videos/{video_id}"
|
||||
# Unmatched: the only template there is the raw path, which may carry a value — so none is sent.
|
||||
assert "X-Route" not in client.get("/api/share/SeCrEt-VaLuE").headers
|
||||
|
||||
|
||||
def test_a_client_supplied_id_is_reused_only_when_it_is_well_formed(sink):
|
||||
client = _client()
|
||||
good = client.get("/api/videos/x", headers={"X-Request-Id": "web-0123456789ab"})
|
||||
assert good.headers["X-Request-Id"] == "web-0123456789ab"
|
||||
# An unbounded header would be a way to write arbitrary text into the log; short/odd ones are
|
||||
# replaced with our own id rather than trusted.
|
||||
for bogus in ("short", "has spaces and \n newline", "x" * 200):
|
||||
r = client.get("/api/videos/x", headers={"X-Request-Id": bogus})
|
||||
assert r.headers["X-Request-Id"] != bogus
|
||||
|
||||
|
||||
def test_the_healthcheck_is_logged_at_debug_so_it_cannot_drown_the_log(sink):
|
||||
http = logging.getLogger("siftlode.http")
|
||||
http.setLevel(logs.TRACE)
|
||||
_client().get("/healthz")
|
||||
assert sink.objects()[-1]["level"] == "DEBUG"
|
||||
# ...but it is still logged when something is wrong with it: the floor drops, nothing is hidden.
|
||||
http.setLevel(logging.INFO)
|
||||
before = len(sink.lines)
|
||||
_client().get("/healthz")
|
||||
assert len(sink.lines) == before
|
||||
|
||||
|
||||
def test_a_failing_request_is_logged_by_status_and_an_unhandled_error_is_logged_at_all(sink):
|
||||
client = _client()
|
||||
client.get("/api/videos/missing/deeper") # 404 from the router
|
||||
assert sink.objects()[-1]["level"] == "WARNING"
|
||||
client.get("/boom")
|
||||
payload = sink.objects()[-1]
|
||||
# The request that most needs a line is the one that raised: uvicorn's access log wrote it, but
|
||||
# our middleware sees the exception instead of a response, so it has to log it explicitly.
|
||||
assert payload["level"] == "ERROR"
|
||||
assert payload["event"] == "http.request"
|
||||
assert "RuntimeError" in payload["exc"]
|
||||
@@ -0,0 +1,260 @@
|
||||
"""E22 S3 — the `log_events` sink: what reaches the table, in what shape, and what it refuses to do.
|
||||
|
||||
The sink is driven synchronously (`write_pending()` on the test's own thread) so nothing here waits on
|
||||
a timer. It is attached to a private logger rather than the root, for the same reason test_logs.py's
|
||||
capture handler is: a `configure_logging` call elsewhere replaces the root handlers.
|
||||
DB lane (`db` fixture): the rows are read back from a real Postgres.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from fastapi import BackgroundTasks
|
||||
from sqlalchemy import create_engine, func, select
|
||||
|
||||
from app import logs, logsink, scheduler
|
||||
from app.auth import purge_user
|
||||
from app.db import engine
|
||||
from app.models import LogEvent
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stored(db):
|
||||
"""A LogSink on the logger `siftlode.sinktest`, and a reader for the rows it wrote."""
|
||||
sink = logsink.LogSink(engine)
|
||||
logger = logging.getLogger("siftlode.sinktest")
|
||||
logger.addHandler(sink)
|
||||
logger.setLevel(logs.TRACE)
|
||||
|
||||
def rows() -> list[LogEvent]:
|
||||
sink.write_pending()
|
||||
db.expire_all()
|
||||
return list(db.execute(select(LogEvent).order_by(LogEvent.id)).scalars())
|
||||
|
||||
try:
|
||||
yield sink, logger, rows
|
||||
finally:
|
||||
logger.removeHandler(sink)
|
||||
|
||||
|
||||
def test_warnings_and_request_lines_are_stored_and_everything_else_is_not(stored):
|
||||
_sink, log, rows = stored
|
||||
log.info("an ordinary info line")
|
||||
log.debug("GET /healthz 200", extra=logs.event("http.request", path="/healthz", status=200))
|
||||
log.info("GET /api/feed 200", extra=logs.event("http.request", path="/api/feed", status=200))
|
||||
log.warning("scanner", extra={**logs.event("http.request", status=404), logs.PERSIST: False})
|
||||
log.warning("disk nearly full")
|
||||
assert [r.msg for r in rows()] == ["GET /api/feed 200", "disk nearly full"]
|
||||
|
||||
|
||||
def test_a_row_is_redacted_exactly_like_the_stdout_line(stored):
|
||||
_sink, log, rows = stored
|
||||
try:
|
||||
raise ValueError("reset link https://x.test/#reset=abc123")
|
||||
except ValueError:
|
||||
log.exception(
|
||||
"sign-in failed for someone@example.com",
|
||||
extra=logs.event("auth.failed", password="hunter2", reason="bad_password"),
|
||||
)
|
||||
(row,) = rows()
|
||||
assert row.level == "ERROR" and row.sev == logs.SEVERITY["ERROR"]
|
||||
assert row.event == "auth.failed"
|
||||
assert "someone@example.com" not in row.msg and logs.REDACTED_EMAIL in row.msg
|
||||
assert row.attrs == {"password": logs.REDACTED, "reason": "bad_password"}
|
||||
assert "abc123" not in row.exc and "ValueError" in row.exc
|
||||
assert row.source == logs.current_source() and row.version
|
||||
|
||||
|
||||
def test_correlation_is_captured_when_the_line_is_logged_not_when_it_is_written(stored):
|
||||
_sink, log, rows = stored
|
||||
with logs.bind(job_id=7, request_id="req-0123456789"):
|
||||
log.warning("inside the job")
|
||||
# The writer runs later, outside the bind — the row still belongs to job 7.
|
||||
(row,) = rows()
|
||||
assert (row.job_id, row.request_id) == (7, "req-0123456789")
|
||||
|
||||
|
||||
def test_a_flood_of_one_message_is_stored_up_to_the_limit_and_then_summarised(stored):
|
||||
sink, log, rows = stored
|
||||
limit = logsink.REPEATS_PER_MINUTE
|
||||
for i in range(limit + 70):
|
||||
log.warning("RSS poll skipped for channel %s: HTTP 404", f"UC{i}")
|
||||
log.warning("a different message")
|
||||
assert len(rows()) == limit + 1 # the flood capped; the other message untouched
|
||||
sink._repeats.started -= logsink.REPEAT_WINDOW_SECONDS # the minute is over
|
||||
notice = rows()[-1]
|
||||
assert notice.event == "log.repeats_suppressed"
|
||||
assert notice.attrs["suppressed"] == 70
|
||||
assert notice.attrs["template"] == "RSS poll skipped for channel %s: HTTP 404"
|
||||
|
||||
|
||||
def test_request_lines_are_never_treated_as_a_flood(stored):
|
||||
_sink, log, rows = stored
|
||||
for _ in range(logsink.REPEATS_PER_MINUTE + 5):
|
||||
log.info("%s %s %s", "GET", "/api/feed", 200, extra=logs.event("http.request"))
|
||||
assert len(rows()) == logsink.REPEATS_PER_MINUTE + 5
|
||||
|
||||
|
||||
def test_a_full_queue_drops_and_says_how_many(db, monkeypatch):
|
||||
monkeypatch.setattr(logsink, "QUEUE_MAX", 3)
|
||||
sink = logsink.LogSink(engine)
|
||||
log = logging.getLogger("siftlode.sinktest.small")
|
||||
log.addHandler(sink)
|
||||
try:
|
||||
for i in range(5):
|
||||
log.warning("message number %d", i)
|
||||
finally:
|
||||
log.removeHandler(sink)
|
||||
assert sink.write_pending() == 4
|
||||
notice = db.execute(select(LogEvent).where(LogEvent.event == "log.sink.dropped")).scalar_one()
|
||||
assert notice.attrs == {"dropped": 2}
|
||||
|
||||
|
||||
def test_an_unreachable_database_costs_the_batch_not_the_caller(capsys):
|
||||
dead = create_engine(
|
||||
"postgresql+psycopg://x:y@127.0.0.1:1/none", connect_args={"connect_timeout": 1}
|
||||
)
|
||||
sink = logsink.LogSink(dead)
|
||||
log = logging.getLogger("siftlode.sinktest.dead")
|
||||
log.addHandler(sink)
|
||||
try:
|
||||
log.error("this has nowhere to go") # must not raise
|
||||
finally:
|
||||
log.removeHandler(sink)
|
||||
assert sink.write_pending() == 0
|
||||
assert "could not store 1 log events" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_the_writer_thread_does_not_store_its_own_records(stored):
|
||||
sink, log, rows = stored
|
||||
sink._writer = threading.current_thread() # as if this line came from inside the writer
|
||||
log.error("a driver warning raised while inserting")
|
||||
sink._writer = None
|
||||
assert rows() == []
|
||||
|
||||
|
||||
def test_gc_keeps_the_retention_window_and_never_keeps_forever(db):
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
def add(age: timedelta) -> None:
|
||||
db.add(
|
||||
LogEvent(ts=now - age, level="WARNING", sev=13, source="api", logger="t", msg=str(age))
|
||||
)
|
||||
|
||||
for age in (timedelta(hours=1), timedelta(days=2), timedelta(days=8)):
|
||||
add(age)
|
||||
db.commit()
|
||||
assert logsink.gc(db, 7) == 1
|
||||
assert logsink.gc(db, 0) == 1 # 0 is not "forever": stored logs always expire (floor: 1 day)
|
||||
db.commit()
|
||||
assert [r.msg for r in db.execute(select(LogEvent)).scalars()] == [str(timedelta(hours=1))]
|
||||
|
||||
|
||||
def test_log_gc_is_a_scheduled_job():
|
||||
assert "log_gc" in scheduler.JOB_FUNCS and scheduler.JOB_INTERVALS["log_gc"] >= 1
|
||||
|
||||
|
||||
def test_deleting_an_account_deletes_its_stored_log_events(db, make_user):
|
||||
gone, kept = make_user(), make_user()
|
||||
for uid in (gone.id, kept.id, gone.id):
|
||||
db.add(
|
||||
LogEvent(
|
||||
ts=datetime.now(timezone.utc),
|
||||
level="INFO",
|
||||
sev=9,
|
||||
source="api",
|
||||
logger="t",
|
||||
msg="x",
|
||||
user_id=uid,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
purge_user(db, gone, BackgroundTasks())
|
||||
assert [r.user_id for r in db.execute(select(LogEvent)).scalars()] == [kept.id]
|
||||
|
||||
|
||||
def test_an_erased_accounts_late_lines_are_stored_without_it(stored, db, make_user):
|
||||
"""The DELETE in purge_user cannot reach a row still on its way: one queued before the erasure,
|
||||
or the erasing request's own access line, logged after the handler committed."""
|
||||
_sink, log, rows = stored
|
||||
user = make_user()
|
||||
# `logs._erased` outlives the test; conftest's `_reset_log_erasures` clears it before each one.
|
||||
log.warning("queued before the erasure", extra={"user_id": user.id})
|
||||
purge_user(db, user, BackgroundTasks())
|
||||
log.info("DELETE /api/me/account 200", extra={**logs.event("http.request"), "user_id": user.id})
|
||||
assert [(r.msg, r.user_id) for r in rows()] == [
|
||||
("queued before the erasure", None),
|
||||
("DELETE /api/me/account 200", None),
|
||||
]
|
||||
|
||||
|
||||
def test_every_path_into_the_table_drops_an_erased_account():
|
||||
"""The two behavioural tests above cover the two writers there are today (this sink and
|
||||
`POST /api/diagnostics/events`). This one covers the THIRD one nobody has written yet: a new
|
||||
`insert(LogEvent)` that forgets the filter would leave an erased account's rows named, and no
|
||||
test of an existing path can notice. Source-level on purpose — it is a rule about call sites."""
|
||||
import pathlib
|
||||
|
||||
root = pathlib.Path(__file__).resolve().parents[1] / "app"
|
||||
unfiltered = [
|
||||
f"{path.relative_to(root)}:{n}"
|
||||
for path in root.rglob("*.py")
|
||||
for n, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1)
|
||||
if "insert(LogEvent)" in line and "without_erased" not in line
|
||||
]
|
||||
assert unfiltered == [], f"insert(LogEvent) without logsink.without_erased at {unfiltered}"
|
||||
|
||||
|
||||
def test_a_nul_character_costs_nothing_but_itself(stored):
|
||||
"""Postgres refuses NUL in text and JSONB, and a refused row fails its whole batch — every other
|
||||
user's rows with it. `GET /assets/%00` logs exactly that."""
|
||||
_sink, log, rows = stored
|
||||
log.warning("before")
|
||||
log.warning("GET /assets/\x00 404", extra=logs.event("http.request", path="/assets/\x00"))
|
||||
log.warning("after")
|
||||
before, nul, after = rows()
|
||||
assert (before.msg, after.msg) == ("before", "after")
|
||||
assert nul.msg == "GET /assets/� 404" and nul.attrs["path"] == "/assets/�"
|
||||
|
||||
|
||||
def test_the_writer_drains_what_was_queued_when_it_is_stopped(db):
|
||||
sink = logsink.LogSink(engine)
|
||||
log = logging.getLogger("siftlode.sinktest.drain")
|
||||
log.addHandler(sink)
|
||||
try:
|
||||
for i in range(logsink.BATCH_MAX + 5):
|
||||
log.warning(f"line {i}") # distinct templates: no flood limiter involved
|
||||
finally:
|
||||
log.removeHandler(sink)
|
||||
sink._stopping.set()
|
||||
sink._run() # stopped before its first wait: everything goes through the drain
|
||||
assert (
|
||||
db.execute(select(func.count()).select_from(LogEvent)).scalar_one() == logsink.BATCH_MAX + 5
|
||||
)
|
||||
|
||||
|
||||
def test_the_running_sink_keeps_the_base_level_for_everyone_the_elevation_is_not_for(
|
||||
db, monkeypatch
|
||||
):
|
||||
"""`logsink.start()` as production runs it. One user's elevation lowers the ROOT logger, so an
|
||||
ordinary request line below the base level reaches every handler — the sink's own level gate is
|
||||
what keeps it out of the table, as stdout's keeps it off the terminal."""
|
||||
root = logging.getLogger()
|
||||
saved = (logs._base_level, dict(logs._elevations), logs._root_applied, root.level)
|
||||
monkeypatch.setattr(logsink.settings, "log_db_sink", True)
|
||||
logs.set_level("WARNING")
|
||||
logs._track_elevation(1, logging.DEBUG, datetime.now(timezone.utc) + timedelta(hours=1))
|
||||
log = logging.getLogger("siftlode.sinktest.live")
|
||||
try:
|
||||
logsink.start()
|
||||
log.info("GET /api/feed 200", extra={**logs.event("http.request"), "user_id": 2})
|
||||
log.warning("a warning")
|
||||
finally:
|
||||
logsink.stop()
|
||||
logs._base_level, logs._root_applied = saved[0], saved[2]
|
||||
logs._elevations.clear()
|
||||
logs._elevations.update(saved[1])
|
||||
root.setLevel(saved[3])
|
||||
assert [r.msg for r in db.execute(select(LogEvent)).scalars()] == ["a warning"]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""A row that is gone answers with a reason the bilingual UI can translate (deep-review round 3).
|
||||
|
||||
The 404 a user actually meets is the one for something deleted in another tab or on another device.
|
||||
`owned_or_404` / `get_or_404` sent it as a bare English string ("Unknown view", "Not found"), and the
|
||||
Hungarian UI toasted it verbatim. The structured-404 fix had gone into one ad-hoc helper instead of
|
||||
the shared one, so every other owner check kept the English. Needs the DB.
|
||||
"""
|
||||
|
||||
from app.auth import require_write_scope
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def test_a_gone_view_tag_or_pin_says_what_is_gone(db, make_user, as_user):
|
||||
client = as_user(make_user())
|
||||
for method, url, what in [
|
||||
("patch", "/api/saved-views/999999", "view"),
|
||||
("delete", "/api/saved-views/999999", "view"),
|
||||
("delete", "/api/tags/999999", "tag"),
|
||||
("delete", "/api/discover/pins/999999", "pin"),
|
||||
]:
|
||||
r = getattr(client, method)(url, **({"json": {"name": "x"}} if method == "patch" else {}))
|
||||
assert r.status_code == 404, (url, r.text)
|
||||
assert r.json()["detail"] == {"code": "not_found", "reason": what}, url
|
||||
|
||||
|
||||
def test_a_gone_channel_video_or_user_says_what_is_gone(db, make_user, as_user):
|
||||
"""The siblings that still hand-wrote `db.get(...) is None → 404 "Unknown …"` (deep-review
|
||||
round 4): the same gone row, but the Hungarian UI showed the English."""
|
||||
admin = make_user(role="admin")
|
||||
client = as_user(admin)
|
||||
for method, url, body, what in [
|
||||
("post", "/api/channels/UCgone000000000000000000/block", None, "channel"),
|
||||
("post", "/api/videos/gone0000000/state", {"status": "watched"}, "video"),
|
||||
(
|
||||
"post",
|
||||
"/api/videos/gone0000000/progress",
|
||||
{"position_seconds": 60, "duration_seconds": 600},
|
||||
"video",
|
||||
),
|
||||
("put", "/api/admin/downloads/quota/999999", {"max_bytes": 1}, "user"),
|
||||
]:
|
||||
r = getattr(client, method)(url, **({"json": body} if body is not None else {}))
|
||||
assert r.status_code == 404, (url, r.text)
|
||||
assert r.json()["detail"] == {"code": "not_found", "reason": what}, url
|
||||
|
||||
|
||||
def test_every_gone_channel_route_says_it_the_same_way(db, make_user, as_user, override_dep):
|
||||
"""The round-4 sweep converted ONE of the five `db.get(Channel, …) is None -> 404 "Unknown
|
||||
channel"` sites (block), and shipped the HU/EN string with it — so the Hungarian UI said it in
|
||||
Hungarian on the Block button and in English on the four beside it (deep-review round 5). The
|
||||
asymmetric fix is the smell this repo bans, so all five are pinned together.
|
||||
|
||||
Literally all five: round 6 found this loop issuing three requests under a docstring claiming
|
||||
five, with explore silently absent and subscribe excused as "covered by the tags route" — which
|
||||
reaches `_user_subscription`, a different site answering `reason: subscription`. Reverting
|
||||
either of those two to its bare English string left the whole lane green."""
|
||||
admin = make_user(role="admin")
|
||||
client = as_user(admin)
|
||||
# Subscribe's own gate (the optional YouTube write scope) sits in FRONT of the lookup, so a
|
||||
# plain account 403s before reaching it. The gate is not what is under test here.
|
||||
override_dep(require_write_scope, lambda: admin)
|
||||
gone = "UCgone000000000000000000"
|
||||
for method, url in [
|
||||
("get", f"/api/channels/{gone}"),
|
||||
("post", f"/api/channels/{gone}/explore"),
|
||||
("post", f"/api/channels/{gone}/reset-backfill"),
|
||||
("post", f"/api/channels/{gone}/subscribe"),
|
||||
("post", f"/api/channels/{gone}/block"),
|
||||
]:
|
||||
r = getattr(client, method)(url)
|
||||
assert r.status_code == 404, (url, r.text)
|
||||
assert r.json()["detail"] == {"code": "not_found", "reason": "channel"}, url
|
||||
|
||||
|
||||
def test_a_gone_subscription_recipient_or_job_says_what_is_gone(db, make_user, as_user):
|
||||
"""The three bare-English 404s the same sweep left behind, which `notifyActionError` printed
|
||||
verbatim into an otherwise Hungarian UI: "Not subscribed to this channel", "No user with that
|
||||
email.", "Unknown job".
|
||||
|
||||
All three, with a request each. Round 6 found this test naming three and issuing two: the
|
||||
recipient site had no test in the whole suite (nor did the `errors.notFound.recipient` keys
|
||||
added beside it), and `run_job_now` rode on its PATCH sibling's coverage."""
|
||||
admin = make_user(role="admin")
|
||||
client = as_user(admin)
|
||||
r = client.post("/api/channels/UCgone000000000000000000/tags", json={"tag_id": 1})
|
||||
assert r.status_code == 404, r.text
|
||||
assert r.json()["detail"] == {"code": "not_found", "reason": "subscription"}
|
||||
for method, url in [
|
||||
("patch", "/api/admin/scheduler/jobs/no_such_job"),
|
||||
("post", "/api/admin/scheduler/jobs/no_such_job/run"),
|
||||
]:
|
||||
kw = {"json": {"interval_minutes": 30}} if method == "patch" else {}
|
||||
r = getattr(client, method)(url, **kw)
|
||||
assert r.status_code == 404, (url, r.text)
|
||||
assert r.json()["detail"] == {"code": "not_found", "reason": "job"}, url
|
||||
# Sharing needs a finished download of one's own to share; the recipient lookup is the next check.
|
||||
asset = MediaAsset(
|
||||
source_kind="url", source_ref="share-404", format_sig="sig", status="ready", ref_count=1
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
job = DownloadJob(
|
||||
user_id=admin.id,
|
||||
asset_id=asset.id,
|
||||
source_kind="url",
|
||||
source_ref="share-404",
|
||||
status="done",
|
||||
queue_pos=1,
|
||||
profile_snapshot={},
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
r = client.post(f"/api/downloads/{job.id}/share", json={"email": "nobody@example.com"})
|
||||
assert r.status_code == 404, r.text
|
||||
assert r.json()["detail"] == {"code": "not_found", "reason": "recipient"}
|
||||
|
||||
|
||||
def test_a_nul_in_a_request_string_is_a_422_not_a_500(db, make_user, as_user):
|
||||
"""Postgres cannot hold NUL in text, so the driver refuses the query outright: a DataError, i.e.
|
||||
a 500 with a traceback, wherever a `%00` or an escaped NUL reached a comparison (deep-review
|
||||
round 4 — round 3 had guarded only the by-rating_key lookup). One app-wide handler now answers
|
||||
all of them."""
|
||||
client = as_user(make_user())
|
||||
r = client.get("/api/plex/playlists?contains=%00")
|
||||
assert r.status_code == 422, r.text
|
||||
assert r.json()["detail"] == {"code": "invalid_input", "reason": "nul"}
|
||||
assert client.get("/api/plex/playlists?contains_group=1,%00").status_code == 422
|
||||
|
||||
|
||||
def test_any_other_data_error_is_still_a_500():
|
||||
"""The other half of the app-wide handler, and the half a refactor can delete in silence: it is
|
||||
registered for EVERY DataError on every route and decides on a substring of the driver's message,
|
||||
so only a NUL may become a 422. Without this, simplifying it to an unconditional 422 — it reads
|
||||
like dead defensive code — would answer a numeric overflow or an over-long ingested title with
|
||||
"your input was bad", losing the traceback that names the real fault (deep-review round 5)."""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.exc import DataError
|
||||
|
||||
from app.main import _nul_in_text
|
||||
|
||||
overflow = DataError("SELECT 1", {}, Exception("numeric field overflow"))
|
||||
with pytest.raises(DataError):
|
||||
asyncio.run(_nul_in_text(None, overflow)) # re-raised: it stays the 500 it always was
|
||||
|
||||
nul = DataError("SELECT 1", {}, Exception("invalid byte sequence: NUL (0x00)"))
|
||||
assert asyncio.run(_nul_in_text(None, nul)).status_code == 422
|
||||
@@ -128,6 +128,60 @@ def test_move_into_someone_elses_list_404s_and_changes_nothing(db, make_user, as
|
||||
assert _ids(db, theirs.id) == []
|
||||
|
||||
|
||||
def test_a_list_that_is_not_yours_answers_with_a_reason_the_ui_can_translate(
|
||||
db, make_user, as_user
|
||||
):
|
||||
# The one 404 a user meets in practice (the list went away in another tab). It used to be a bare
|
||||
# English "Unknown playlist", which the Hungarian UI then showed verbatim.
|
||||
user, other = make_user(), make_user()
|
||||
_seed_videos(db, ["v1"])
|
||||
theirs = _playlist(db, other, "Theirs", ["v1"])
|
||||
|
||||
r = as_user(user).delete(f"/api/playlists/{theirs.id}/items/v1")
|
||||
assert r.status_code == 404
|
||||
assert r.json()["detail"] == {"code": "not_found", "reason": "playlist"}
|
||||
db.expire_all()
|
||||
assert _ids(db, theirs.id) == ["v1"]
|
||||
|
||||
|
||||
def test_a_move_into_a_list_that_is_gone_names_the_target_not_the_list_on_screen(
|
||||
db, make_user, as_user
|
||||
):
|
||||
# Both lists 404'd with the same reason, which the UI reads as "THIS playlist doesn't exist any
|
||||
# more" — said about the source the reader is looking at, which is still there.
|
||||
user = make_user()
|
||||
_seed_videos(db, ["v1"])
|
||||
src = _playlist(db, user, "Mine", ["v1"])
|
||||
gone = _playlist(db, user, "Gone", [])
|
||||
gone_id = gone.id
|
||||
db.delete(gone)
|
||||
db.commit()
|
||||
|
||||
r = _move(as_user(user), src.id, "v1", gone_id)
|
||||
assert r.status_code == 404
|
||||
assert r.json()["detail"] == {"code": "not_found", "reason": "move_target"}
|
||||
db.expire_all()
|
||||
assert _ids(db, src.id) == ["v1"]
|
||||
|
||||
|
||||
def test_a_refused_playlist_action_answers_with_a_reason_the_ui_can_translate(
|
||||
db, make_user, as_user
|
||||
):
|
||||
# The rest of the router's refusals were bare English too ("The Watch later list can't be
|
||||
# deleted.", "YouTube delete failed.", ...), shown verbatim under the Hungarian UI.
|
||||
user = make_user()
|
||||
client = as_user(user)
|
||||
wl = client.post("/api/playlists/watch-later", json={"video_id": "nope"})
|
||||
assert wl.status_code == 404
|
||||
assert wl.json()["detail"] == {"code": "not_found", "reason": "video"}
|
||||
_seed_videos(db, ["v1"])
|
||||
saved = client.post("/api/playlists/watch-later", json={"video_id": "v1"})
|
||||
wl_id = saved.json()["playlist_id"]
|
||||
r = client.delete(f"/api/playlists/{wl_id}")
|
||||
assert r.status_code == 400
|
||||
assert r.json()["detail"] == {"code": "playlist", "reason": "watch_later_undeletable"}
|
||||
|
||||
|
||||
def test_move_marks_both_linked_sides_dirty(db, make_user, as_user):
|
||||
"""A move edits two YouTube mirrors at once — the source lost an item, the target gained one.
|
||||
Marking only the source is the asymmetric fix this repo keeps re-learning."""
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
""" "Clear resume point" has to reach Plex — the case the status-only guard skipped.
|
||||
|
||||
`POST /api/plex/item/{rk}/state {"status": "new"}` backs the *Clear resume point* / *Mark unwatched*
|
||||
buttons. It deletes the local row and mirrors the un-mark to Plex — but it used to decide whether to
|
||||
push from the PREVIOUS STATUS alone (`prev_status != "new"`), and an in-progress item's row is
|
||||
`status="new"` carrying a position: that is how the importer maps a bare `viewOffset`
|
||||
(`_plex_watch_to_state`). So for the one case the button exists for, nothing was ever sent to Plex,
|
||||
Plex kept the offset, and the next pull imported it straight back — the user cleared *Protector*
|
||||
twice and it returned to Continue watching at 92s both times (2026-09-09).
|
||||
|
||||
One `unscrobble` clears the view count AND the offset (measured against the live server), so the
|
||||
push itself needed no change; what needed fixing is WHEN it is scheduled.
|
||||
|
||||
DB lane (`db` fixture) — skipped when no Postgres is reachable.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import PlexItem, PlexLibrary, PlexState, User
|
||||
from app.plex import watch_sync as plex_watch
|
||||
|
||||
|
||||
def _seed(db, *, status: str, position: int):
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
it = PlexItem(
|
||||
rating_key="98614", library_id=lib.id, kind="movie", title="Protector", duration_s=5495
|
||||
)
|
||||
db.add(it)
|
||||
user = User(email="clear@example.com")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
db.add(PlexState(user_id=user.id, item_id=it.id, status=status, position_seconds=position))
|
||||
db.commit()
|
||||
return it, user
|
||||
|
||||
|
||||
def _clear(db, user, it, monkeypatch) -> list[tuple]:
|
||||
"""Drive the endpoint with the Plex link satisfied and the background push recorded."""
|
||||
pushes: list[tuple] = []
|
||||
monkeypatch.setattr(state, "_configured_cache", True) # pass the instance-setup request gate
|
||||
monkeypatch.setattr(plex_watch, "link_for_push", lambda db_, uid: SimpleNamespace(id=1))
|
||||
monkeypatch.setattr(
|
||||
plex_watch,
|
||||
"push_state_to_plex",
|
||||
lambda user_id, item_id, rating_key, action, *rest: pushes.append((rating_key, action)),
|
||||
)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
resp = TestClient(app).post(f"/api/plex/item/{it.rating_key}/state", json={"status": "new"})
|
||||
assert resp.status_code == 200, resp.text
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
return pushes
|
||||
|
||||
|
||||
def test_clearing_an_in_progress_item_tells_plex(db, monkeypatch):
|
||||
# The regression: status "new" + a position IS the in-progress shape, and it must push.
|
||||
it, user = _seed(db, status="new", position=92)
|
||||
assert _clear(db, user, it, monkeypatch) == [("98614", "unwatched")]
|
||||
assert db.query(PlexState).count() == 0
|
||||
|
||||
|
||||
def test_clearing_a_watched_item_still_tells_plex(db, monkeypatch):
|
||||
# The case the old guard did cover — kept, so the fix cannot narrow into a new hole.
|
||||
it, user = _seed(db, status="watched", position=0)
|
||||
assert _clear(db, user, it, monkeypatch) == [("98614", "unwatched")]
|
||||
|
||||
|
||||
def test_unhiding_touches_plex_not_at_all(db, monkeypatch):
|
||||
# `hidden` is a Siftlode-only concept "with no Plex analogue, so it never touches Plex" — the
|
||||
# endpoint's own docstring. The old status-only guard pushed here anyway, un-marking an item on
|
||||
# Plex that the user had merely hidden.
|
||||
it, user = _seed(db, status="hidden", position=0)
|
||||
assert _clear(db, user, it, monkeypatch) == []
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user